diff --git a/api/allocations.go b/api/allocations.go index dac3e2bd2713..8d02791273b1 100644 --- a/api/allocations.go +++ b/api/allocations.go @@ -143,6 +143,8 @@ type AllocationListStub struct { // healthy. type AllocDeploymentStatus struct { Healthy *bool + Timestamp time.Time + Canary bool ModifyIndex uint64 } @@ -214,6 +216,10 @@ type DesiredTransition struct { // Migrate is used to indicate that this allocation should be stopped and // migrated to another node. Migrate *bool + + // Reschedule is used to indicate that this allocation is eligible to be + // rescheduled. + Reschedule *bool } // ShouldMigrate returns whether the transition object dictates a migration. diff --git a/api/deployments.go b/api/deployments.go index 0b996f73c03a..7910907742bc 100644 --- a/api/deployments.go +++ b/api/deployments.go @@ -2,6 +2,7 @@ package api import ( "sort" + "time" ) // Deployments is used to query the deployments endpoints. @@ -139,14 +140,16 @@ type Deployment struct { // DeploymentState tracks the state of a deployment for a given task group. type DeploymentState struct { - PlacedCanaries []string - AutoRevert bool - Promoted bool - DesiredCanaries int - DesiredTotal int - PlacedAllocs int - HealthyAllocs int - UnhealthyAllocs int + PlacedCanaries []string + AutoRevert bool + ProgressDeadline time.Duration + RequireProgressBy time.Time + Promoted bool + DesiredCanaries int + DesiredTotal int + PlacedAllocs int + HealthyAllocs int + UnhealthyAllocs int } // DeploymentIndexSort is a wrapper to sort deployments by CreateIndex. We diff --git a/api/jobs.go b/api/jobs.go index b56b9c8646bb..d3a255d06b8c 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -343,26 +343,28 @@ type periodicForceResponse struct { // UpdateStrategy defines a task groups update strategy. type UpdateStrategy struct { - Stagger *time.Duration `mapstructure:"stagger"` - MaxParallel *int `mapstructure:"max_parallel"` - HealthCheck *string `mapstructure:"health_check"` - MinHealthyTime *time.Duration `mapstructure:"min_healthy_time"` - HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"` - AutoRevert *bool `mapstructure:"auto_revert"` - Canary *int `mapstructure:"canary"` + Stagger *time.Duration `mapstructure:"stagger"` + MaxParallel *int `mapstructure:"max_parallel"` + HealthCheck *string `mapstructure:"health_check"` + MinHealthyTime *time.Duration `mapstructure:"min_healthy_time"` + HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"` + ProgressDeadline *time.Duration `mapstructure:"progress_deadline"` + AutoRevert *bool `mapstructure:"auto_revert"` + Canary *int `mapstructure:"canary"` } // DefaultUpdateStrategy provides a baseline that can be used to upgrade // jobs with the old policy or for populating field defaults. func DefaultUpdateStrategy() *UpdateStrategy { return &UpdateStrategy{ - Stagger: helper.TimeToPtr(30 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(5 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(30 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(5 * time.Minute), + ProgressDeadline: helper.TimeToPtr(10 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), } } @@ -393,6 +395,10 @@ func (u *UpdateStrategy) Copy() *UpdateStrategy { copy.HealthyDeadline = helper.TimeToPtr(*u.HealthyDeadline) } + if u.ProgressDeadline != nil { + copy.ProgressDeadline = helper.TimeToPtr(*u.ProgressDeadline) + } + if u.AutoRevert != nil { copy.AutoRevert = helper.BoolToPtr(*u.AutoRevert) } @@ -429,6 +435,10 @@ func (u *UpdateStrategy) Merge(o *UpdateStrategy) { u.HealthyDeadline = helper.TimeToPtr(*o.HealthyDeadline) } + if o.ProgressDeadline != nil { + u.ProgressDeadline = helper.TimeToPtr(*o.ProgressDeadline) + } + if o.AutoRevert != nil { u.AutoRevert = helper.BoolToPtr(*o.AutoRevert) } @@ -457,6 +467,10 @@ func (u *UpdateStrategy) Canonicalize() { u.HealthyDeadline = d.HealthyDeadline } + if u.ProgressDeadline == nil { + u.ProgressDeadline = d.ProgressDeadline + } + if u.MinHealthyTime == nil { u.MinHealthyTime = d.MinHealthyTime } @@ -496,6 +510,10 @@ func (u *UpdateStrategy) Empty() bool { return false } + if u.ProgressDeadline != nil && *u.ProgressDeadline != 0 { + return false + } + if u.AutoRevert != nil && *u.AutoRevert { return false } diff --git a/api/jobs_test.go b/api/jobs_test.go index 9119f511835d..dfefde3b41e0 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -304,9 +304,10 @@ func TestJobs_Canonicalize(t *testing.T) { }, Services: []*Service{ { - Name: "redis-cache", - Tags: []string{"global", "cache"}, - PortLabel: "db", + Name: "redis-cache", + Tags: []string{"global", "cache"}, + CanaryTags: []string{"canary", "global", "cache"}, + PortLabel: "db", Checks: []ServiceCheck{ { Name: "alive", @@ -354,13 +355,14 @@ func TestJobs_Canonicalize(t *testing.T) { JobModifyIndex: helper.Uint64ToPtr(0), Datacenters: []string{"dc1"}, Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(30 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(5 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(30 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(5 * time.Minute), + ProgressDeadline: helper.TimeToPtr(10 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), }, TaskGroups: []*TaskGroup{ { @@ -387,13 +389,14 @@ func TestJobs_Canonicalize(t *testing.T) { }, Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(30 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(5 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(30 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(5 * time.Minute), + ProgressDeadline: helper.TimeToPtr(10 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), }, Migrate: DefaultMigrateStrategy(), Tasks: []*Task{ @@ -425,6 +428,7 @@ func TestJobs_Canonicalize(t *testing.T) { { Name: "redis-cache", Tags: []string{"global", "cache"}, + CanaryTags: []string{"canary", "global", "cache"}, PortLabel: "db", AddressMode: "auto", Checks: []ServiceCheck{ @@ -515,13 +519,14 @@ func TestJobs_Canonicalize(t *testing.T) { ID: helper.StringToPtr("bar"), ParentID: helper.StringToPtr("lol"), Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(1 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(6 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(1 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(6 * time.Minute), + ProgressDeadline: helper.TimeToPtr(7 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), }, TaskGroups: []*TaskGroup{ { @@ -569,13 +574,14 @@ func TestJobs_Canonicalize(t *testing.T) { ModifyIndex: helper.Uint64ToPtr(0), JobModifyIndex: helper.Uint64ToPtr(0), Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(1 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(6 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(1 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(6 * time.Minute), + ProgressDeadline: helper.TimeToPtr(7 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), }, TaskGroups: []*TaskGroup{ { @@ -601,13 +607,14 @@ func TestJobs_Canonicalize(t *testing.T) { Unlimited: helper.BoolToPtr(true), }, Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(2 * time.Second), - MaxParallel: helper.IntToPtr(2), - HealthCheck: helper.StringToPtr("manual"), - MinHealthyTime: helper.TimeToPtr(1 * time.Second), - HealthyDeadline: helper.TimeToPtr(6 * time.Minute), - AutoRevert: helper.BoolToPtr(true), - Canary: helper.IntToPtr(1), + Stagger: helper.TimeToPtr(2 * time.Second), + MaxParallel: helper.IntToPtr(2), + HealthCheck: helper.StringToPtr("manual"), + MinHealthyTime: helper.TimeToPtr(1 * time.Second), + HealthyDeadline: helper.TimeToPtr(6 * time.Minute), + ProgressDeadline: helper.TimeToPtr(7 * time.Minute), + AutoRevert: helper.BoolToPtr(true), + Canary: helper.IntToPtr(1), }, Migrate: DefaultMigrateStrategy(), Tasks: []*Task{ @@ -642,13 +649,14 @@ func TestJobs_Canonicalize(t *testing.T) { Unlimited: helper.BoolToPtr(true), }, Update: &UpdateStrategy{ - Stagger: helper.TimeToPtr(1 * time.Second), - MaxParallel: helper.IntToPtr(1), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(6 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), + Stagger: helper.TimeToPtr(1 * time.Second), + MaxParallel: helper.IntToPtr(1), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(6 * time.Minute), + ProgressDeadline: helper.TimeToPtr(7 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), }, Migrate: DefaultMigrateStrategy(), Tasks: []*Task{ diff --git a/api/tasks.go b/api/tasks.go index e6cb9863e8bb..4c6d3a502b48 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -295,8 +295,9 @@ type Service struct { Id string Name string Tags []string - PortLabel string `mapstructure:"port"` - AddressMode string `mapstructure:"address_mode"` + CanaryTags []string `mapstructure:"canary_tags"` + PortLabel string `mapstructure:"port"` + AddressMode string `mapstructure:"address_mode"` Checks []ServiceCheck CheckRestart *CheckRestart `mapstructure:"check_restart"` } diff --git a/api/tasks_test.go b/api/tasks_test.go index 3ce2a7f669b5..76ebdf9ae263 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -252,13 +252,14 @@ func TestTaskGroup_Canonicalize_Update(t *testing.T) { job := &Job{ ID: helper.StringToPtr("test"), Update: &UpdateStrategy{ - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(0), - HealthCheck: helper.StringToPtr(""), - HealthyDeadline: helper.TimeToPtr(0), - MaxParallel: helper.IntToPtr(0), - MinHealthyTime: helper.TimeToPtr(0), - Stagger: helper.TimeToPtr(0), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(0), + HealthCheck: helper.StringToPtr(""), + HealthyDeadline: helper.TimeToPtr(0), + ProgressDeadline: helper.TimeToPtr(0), + MaxParallel: helper.IntToPtr(0), + MinHealthyTime: helper.TimeToPtr(0), + Stagger: helper.TimeToPtr(0), }, } job.Canonicalize() diff --git a/client/alloc_runner.go b/client/alloc_runner.go index 28ccd00ccb08..aeb284a3b9bd 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -50,7 +50,8 @@ type AllocRunner struct { alloc *structs.Allocation allocClientStatus string // Explicit status of allocation. Set when there are failures allocClientDescription string - allocHealth *bool // Whether the allocation is healthy + allocHealth *bool // Whether the allocation is healthy + allocHealthTime time.Time // Time at which allocation health has been set allocBroadcast *cstructs.AllocBroadcaster allocLock sync.Mutex @@ -580,6 +581,7 @@ func (r *AllocRunner) Alloc() *structs.Allocation { alloc.DeploymentStatus = &structs.AllocDeploymentStatus{} } alloc.DeploymentStatus.Healthy = helper.BoolToPtr(*r.allocHealth) + alloc.DeploymentStatus.Timestamp = r.allocHealthTime } r.allocLock.Unlock() @@ -943,6 +945,7 @@ OUTER: // If the deployment ids have changed clear the health if r.alloc.DeploymentID != update.DeploymentID { r.allocHealth = nil + r.allocHealthTime = time.Time{} } r.alloc = update diff --git a/client/alloc_runner_health_watcher.go b/client/alloc_runner_health_watcher.go index f193b41e2920..ede4eaeaa56b 100644 --- a/client/alloc_runner_health_watcher.go +++ b/client/alloc_runner_health_watcher.go @@ -112,6 +112,7 @@ func (r *AllocRunner) watchHealth(ctx context.Context) { r.allocLock.Lock() r.allocHealth = helper.BoolToPtr(allocHealthy) + r.allocHealthTime = time.Now() r.allocLock.Unlock() // If deployment is unhealthy emit task events explaining why diff --git a/client/consul.go b/client/consul.go index 02e40ef0f09f..58f75e6f8c53 100644 --- a/client/consul.go +++ b/client/consul.go @@ -1,17 +1,14 @@ package client import ( - "github.com/hashicorp/nomad/client/driver" - cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/command/agent/consul" - "github.com/hashicorp/nomad/nomad/structs" ) // ConsulServiceAPI is the interface the Nomad Client uses to register and // remove services and checks from Consul. type ConsulServiceAPI interface { - RegisterTask(allocID string, task *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error - RemoveTask(allocID string, task *structs.Task) - UpdateTask(allocID string, existing, newTask *structs.Task, restart consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error + RegisterTask(*consul.TaskServices) error + RemoveTask(*consul.TaskServices) + UpdateTask(old, newTask *consul.TaskServices) error AllocRegistrations(allocID string) (*consul.AllocRegistration, error) } diff --git a/client/consul_testing.go b/client/consul_testing.go index 4a2d2631bc61..1db5d4ccfa77 100644 --- a/client/consul_testing.go +++ b/client/consul_testing.go @@ -5,11 +5,8 @@ import ( "log" "sync" - "github.com/hashicorp/nomad/client/driver" - cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/go-testing-interface" ) @@ -17,12 +14,10 @@ import ( type mockConsulOp struct { op string // add, remove, or update allocID string - task *structs.Task - exec driver.ScriptExecutor - net *cstructs.DriverNetwork + task string } -func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) mockConsulOp { +func newMockConsulOp(op, allocID, task string) mockConsulOp { if op != "add" && op != "remove" && op != "update" && op != "alloc_registrations" { panic(fmt.Errorf("invalid consul op: %s", op)) } @@ -30,8 +25,6 @@ func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptE op: op, allocID: allocID, task: task, - exec: exec, - net: net, } } @@ -56,34 +49,34 @@ func newMockConsulServiceClient(t testing.T) *mockConsulServiceClient { return &m } -func (m *mockConsulServiceClient) UpdateTask(allocID string, old, new *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error { +func (m *mockConsulServiceClient) UpdateTask(old, new *consul.TaskServices) error { m.mu.Lock() defer m.mu.Unlock() - m.logger.Printf("[TEST] mock_consul: UpdateTask(%q, %v, %v, %T, %x)", allocID, old, new, exec, net.Hash()) - m.ops = append(m.ops, newMockConsulOp("update", allocID, new, exec, net)) + m.logger.Printf("[TEST] mock_consul: UpdateTask(alloc: %s, task: %s)", new.AllocID[:6], new.Name) + m.ops = append(m.ops, newMockConsulOp("update", new.AllocID, new.Name)) return nil } -func (m *mockConsulServiceClient) RegisterTask(allocID string, task *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error { +func (m *mockConsulServiceClient) RegisterTask(task *consul.TaskServices) error { m.mu.Lock() defer m.mu.Unlock() - m.logger.Printf("[TEST] mock_consul: RegisterTask(%q, %q, %T, %x)", allocID, task.Name, exec, net.Hash()) - m.ops = append(m.ops, newMockConsulOp("add", allocID, task, exec, net)) + m.logger.Printf("[TEST] mock_consul: RegisterTask(alloc: %s, task: %s)", task.AllocID, task.Name) + m.ops = append(m.ops, newMockConsulOp("add", task.AllocID, task.Name)) return nil } -func (m *mockConsulServiceClient) RemoveTask(allocID string, task *structs.Task) { +func (m *mockConsulServiceClient) RemoveTask(task *consul.TaskServices) { m.mu.Lock() defer m.mu.Unlock() - m.logger.Printf("[TEST] mock_consul: RemoveTask(%q, %q)", allocID, task.Name) - m.ops = append(m.ops, newMockConsulOp("remove", allocID, task, nil, nil)) + m.logger.Printf("[TEST] mock_consul: RemoveTask(%q, %q)", task.AllocID, task.Name) + m.ops = append(m.ops, newMockConsulOp("remove", task.AllocID, task.Name)) } func (m *mockConsulServiceClient) AllocRegistrations(allocID string) (*consul.AllocRegistration, error) { m.mu.Lock() defer m.mu.Unlock() m.logger.Printf("[TEST] mock_consul: AllocRegistrations(%q)", allocID) - m.ops = append(m.ops, newMockConsulOp("alloc_registrations", allocID, nil, nil, nil)) + m.ops = append(m.ops, newMockConsulOp("alloc_registrations", allocID, "")) if m.allocRegistrationsFn != nil { return m.allocRegistrationsFn(allocID) diff --git a/client/task_runner.go b/client/task_runner.go index ef893436a1f8..4affba3e6b34 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -25,6 +25,7 @@ import ( "github.com/hashicorp/nomad/client/driver" "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/client/vaultclient" + "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/nomad/structs" "github.com/ugorji/go/codec" @@ -1218,8 +1219,7 @@ func (r *TaskRunner) run() { // Remove from consul before killing the task so that traffic // can be rerouted - interpTask := interpolateServices(r.envBuilder.Build(), r.task) - r.consul.RemoveTask(r.alloc.ID, interpTask) + r.removeServices() // Delay actually killing the task if configured. See #244 if r.task.ShutdownDelay > 0 { @@ -1274,8 +1274,7 @@ func (r *TaskRunner) run() { // stopping. Errors are logged. func (r *TaskRunner) cleanup() { // Remove from Consul - interpTask := interpolateServices(r.envBuilder.Build(), r.task) - r.consul.RemoveTask(r.alloc.ID, interpTask) + r.removeServices() drv, err := r.createDriver() if err != nil { @@ -1338,8 +1337,7 @@ func (r *TaskRunner) shouldRestart() bool { } // Unregister from Consul while waiting to restart. - interpTask := interpolateServices(r.envBuilder.Build(), r.task) - r.consul.RemoveTask(r.alloc.ID, interpTask) + r.removeServices() // Sleep but watch for destroy events. select { @@ -1498,7 +1496,8 @@ func (r *TaskRunner) registerServices(d driver.Driver, h driver.DriverHandle, n exec = h } interpolatedTask := interpolateServices(r.envBuilder.Build(), r.task) - return r.consul.RegisterTask(r.alloc.ID, interpolatedTask, r, exec, n) + taskServices := consul.NewTaskServices(r.alloc, interpolatedTask, r, exec, n) + return r.consul.RegisterTask(taskServices) } // interpolateServices interpolates tags in a service and checks with values from the @@ -1532,6 +1531,7 @@ func interpolateServices(taskEnv *env.TaskEnv, task *structs.Task) *structs.Task service.Name = taskEnv.ReplaceEnv(service.Name) service.PortLabel = taskEnv.ReplaceEnv(service.PortLabel) service.Tags = taskEnv.ParseAndReplace(service.Tags) + service.CanaryTags = taskEnv.ParseAndReplace(service.CanaryTags) } return taskCopy } @@ -1679,7 +1679,7 @@ func (r *TaskRunner) handleUpdate(update *structs.Allocation) error { // Update services in Consul newInterpolatedTask := interpolateServices(r.envBuilder.Build(), updatedTask) - if err := r.updateServices(drv, r.handle, oldInterpolatedTask, newInterpolatedTask); err != nil { + if err := r.updateServices(drv, r.handle, r.alloc, oldInterpolatedTask, update, newInterpolatedTask); err != nil { mErr.Errors = append(mErr.Errors, fmt.Errorf("error updating services and checks in Consul: %v", err)) } } @@ -1697,7 +1697,10 @@ func (r *TaskRunner) handleUpdate(update *structs.Allocation) error { } // updateServices and checks with Consul. Tasks must be interpolated! -func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor, oldTask, newTask *structs.Task) error { +func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor, + oldAlloc *structs.Allocation, oldTask *structs.Task, + newAlloc *structs.Allocation, newTask *structs.Task) error { + var exec driver.ScriptExecutor if d.Abilities().Exec { // Allow set the script executor if the driver supports it @@ -1706,7 +1709,23 @@ func (r *TaskRunner) updateServices(d driver.Driver, h driver.ScriptExecutor, ol r.driverNetLock.Lock() net := r.driverNet.Copy() r.driverNetLock.Unlock() - return r.consul.UpdateTask(r.alloc.ID, oldTask, newTask, r, exec, net) + oldTaskServices := consul.NewTaskServices(oldAlloc, oldTask, r, exec, net) + newTaskServices := consul.NewTaskServices(newAlloc, newTask, r, exec, net) + return r.consul.UpdateTask(oldTaskServices, newTaskServices) +} + +// removeServices and checks from Consul. Handles interpolation and deleting +// Canary=true and Canary=false versions in case Canary=false is set at the +// same time as the alloc is stopped. +func (r *TaskRunner) removeServices() { + interpTask := interpolateServices(r.envBuilder.Build(), r.task) + taskServices := consul.NewTaskServices(r.alloc, interpTask, r, nil, nil) + r.consul.RemoveTask(taskServices) + + // Flip Canary and remove again in case canary is getting flipped at + // the same time as the alloc is being destroyed + taskServices.Canary = !taskServices.Canary + r.consul.RemoveTask(taskServices) } // handleDestroy kills the task handle. In the case that killing fails, diff --git a/client/task_runner_test.go b/client/task_runner_test.go index 1e2b5dd358be..1c5dae6bbad5 100644 --- a/client/task_runner_test.go +++ b/client/task_runner_test.go @@ -650,7 +650,7 @@ func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { defer ctx.Cleanup() // Assert it is properly registered and unregistered - if expected := 4; len(consul.ops) != expected { + if expected := 6; len(consul.ops) != expected { t.Errorf("expected %d consul ops but found: %d", expected, len(consul.ops)) } if consul.ops[0].op != "add" { @@ -659,11 +659,17 @@ func TestTaskRunner_UnregisterConsul_Retries(t *testing.T) { if consul.ops[1].op != "remove" { t.Errorf("expected second op to be remove but found: %q", consul.ops[1].op) } - if consul.ops[2].op != "add" { - t.Errorf("expected third op to be add but found: %q", consul.ops[2].op) + if consul.ops[2].op != "remove" { + t.Errorf("expected third op to be remove but found: %q", consul.ops[2].op) } - if consul.ops[3].op != "remove" { - t.Errorf("expected fourth/final op to be remove but found: %q", consul.ops[3].op) + if consul.ops[3].op != "add" { + t.Errorf("expected fourth op to be add but found: %q", consul.ops[3].op) + } + if consul.ops[4].op != "remove" { + t.Errorf("expected fifth op to be remove but found: %q", consul.ops[4].op) + } + if consul.ops[5].op != "remove" { + t.Errorf("expected sixth op to be remove but found: %q", consul.ops[5].op) } } diff --git a/command/agent/consul/client.go b/command/agent/consul/client.go index 660a2e876de7..4c8498128436 100644 --- a/command/agent/consul/client.go +++ b/command/agent/consul/client.go @@ -14,7 +14,6 @@ import ( metrics "github.com/armon/go-metrics" "github.com/hashicorp/consul/api" - "github.com/hashicorp/nomad/client/driver" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" @@ -603,11 +602,11 @@ func (c *ServiceClient) RegisterAgent(role string, services []*structs.Service) // serviceRegs creates service registrations, check registrations, and script // checks from a service. It returns a service registration object with the // service and check IDs populated. -func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *structs.Service, - task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) (*ServiceRegistration, error) { +func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, task *TaskServices) ( + *ServiceRegistration, error) { // Get the services ID - id := makeTaskServiceID(allocID, task.Name, service) + id := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary) sreg := &ServiceRegistration{ serviceID: id, checkIDs: make(map[string]struct{}, len(service.Checks)), @@ -620,26 +619,33 @@ func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *st } // Determine the address to advertise based on the mode - ip, port, err := getAddress(addrMode, service.PortLabel, task.Resources.Networks, net) + ip, port, err := getAddress(addrMode, service.PortLabel, task.Networks, task.DriverNetwork) if err != nil { return nil, fmt.Errorf("unable to get address for service %q: %v", service.Name, err) } + // Determine whether to use tags or canary_tags + var tags []string + if task.Canary { + tags = make([]string, len(service.CanaryTags)) + copy(tags, service.CanaryTags) + } else { + tags = make([]string, len(service.Tags)) + copy(tags, service.Tags) + } + // Build the Consul Service registration request serviceReg := &api.AgentServiceRegistration{ ID: id, Name: service.Name, - Tags: make([]string, len(service.Tags)), + Tags: tags, Address: ip, Port: port, } - // copy isn't strictly necessary but can avoid bugs especially - // with tests that may reuse Tasks - copy(serviceReg.Tags, service.Tags) ops.regServices = append(ops.regServices, serviceReg) // Build the check registrations - checkIDs, err := c.checkRegs(ops, allocID, id, service, task, exec, net) + checkIDs, err := c.checkRegs(ops, id, service, task) if err != nil { return nil, err } @@ -651,8 +657,8 @@ func (c *ServiceClient) serviceRegs(ops *operations, allocID string, service *st // checkRegs registers the checks for the given service and returns the // registered check ids. -func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, service *structs.Service, - task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) ([]string, error) { +func (c *ServiceClient) checkRegs(ops *operations, serviceID string, service *structs.Service, + task *TaskServices) ([]string, error) { // Fast path numChecks := len(service.Checks) @@ -665,11 +671,13 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se checkID := makeCheckID(serviceID, check) checkIDs = append(checkIDs, checkID) if check.Type == structs.ServiceCheckScript { - if exec == nil { + if task.DriverExec == nil { return nil, fmt.Errorf("driver doesn't support script checks") } - ops.scripts = append(ops.scripts, newScriptCheck( - allocID, task.Name, checkID, check, exec, c.client, c.logger, c.shutdownCh)) + + sc := newScriptCheck(task.AllocID, task.Name, checkID, check, task.DriverExec, + c.client, c.logger, c.shutdownCh) + ops.scripts = append(ops.scripts, sc) // Skip getAddress for script checks checkReg, err := createCheckReg(serviceID, checkID, check, "", 0) @@ -693,7 +701,7 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se addrMode = structs.AddressModeHost } - ip, port, err := getAddress(addrMode, portLabel, task.Resources.Networks, net) + ip, port, err := getAddress(addrMode, portLabel, task.Networks, task.DriverNetwork) if err != nil { return nil, fmt.Errorf("error getting address for check %q: %v", check.Name, err) } @@ -714,7 +722,7 @@ func (c *ServiceClient) checkRegs(ops *operations, allocID, serviceID string, se // Checks will always use the IP from the Task struct (host's IP). // // Actual communication with Consul is done asynchronously (see Run). -func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error { +func (c *ServiceClient) RegisterTask(task *TaskServices) error { // Fast path numServices := len(task.Services) if numServices == 0 { @@ -726,7 +734,7 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart ops := &operations{} for _, service := range task.Services { - sreg, err := c.serviceRegs(ops, allocID, service, task, exec, net) + sreg, err := c.serviceRegs(ops, service, task) if err != nil { return err } @@ -734,18 +742,18 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart } // Add the task to the allocation's registration - c.addTaskRegistration(allocID, task.Name, t) + c.addTaskRegistration(task.AllocID, task.Name, t) c.commit(ops) // Start watching checks. Done after service registrations are built // since an error building them could leak watches. for _, service := range task.Services { - serviceID := makeTaskServiceID(allocID, task.Name, service) + serviceID := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary) for _, check := range service.Checks { if check.TriggersRestarts() { checkID := makeCheckID(serviceID, check) - c.checkWatcher.Watch(allocID, task.Name, checkID, check, restarter) + c.checkWatcher.Watch(task.AllocID, task.Name, checkID, check, task.Restarter) } } } @@ -756,19 +764,19 @@ func (c *ServiceClient) RegisterTask(allocID string, task *structs.Task, restart // changed. // // DriverNetwork must not change between invocations for the same allocation. -func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error { +func (c *ServiceClient) UpdateTask(old, newTask *TaskServices) error { ops := &operations{} taskReg := new(TaskRegistration) taskReg.Services = make(map[string]*ServiceRegistration, len(newTask.Services)) - existingIDs := make(map[string]*structs.Service, len(existing.Services)) - for _, s := range existing.Services { - existingIDs[makeTaskServiceID(allocID, existing.Name, s)] = s + existingIDs := make(map[string]*structs.Service, len(old.Services)) + for _, s := range old.Services { + existingIDs[makeTaskServiceID(old.AllocID, old.Name, s, old.Canary)] = s } newIDs := make(map[string]*structs.Service, len(newTask.Services)) for _, s := range newTask.Services { - newIDs[makeTaskServiceID(allocID, newTask.Name, s)] = s + newIDs[makeTaskServiceID(newTask.AllocID, newTask.Name, s, newTask.Canary)] = s } // Loop over existing Service IDs to see if they have been removed or @@ -816,7 +824,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta } // New check on an unchanged service; add them now - newCheckIDs, err := c.checkRegs(ops, allocID, existingID, newSvc, newTask, exec, net) + newCheckIDs, err := c.checkRegs(ops, existingID, newSvc, newTask) if err != nil { return err } @@ -828,7 +836,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta // Update all watched checks as CheckRestart fields aren't part of ID if check.TriggersRestarts() { - c.checkWatcher.Watch(allocID, newTask.Name, checkID, check, restarter) + c.checkWatcher.Watch(newTask.AllocID, newTask.Name, checkID, check, newTask.Restarter) } } @@ -845,7 +853,7 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta // Any remaining services should just be enqueued directly for _, newSvc := range newIDs { - sreg, err := c.serviceRegs(ops, allocID, newSvc, newTask, exec, net) + sreg, err := c.serviceRegs(ops, newSvc, newTask) if err != nil { return err } @@ -854,18 +862,18 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta } // Add the task to the allocation's registration - c.addTaskRegistration(allocID, newTask.Name, taskReg) + c.addTaskRegistration(newTask.AllocID, newTask.Name, taskReg) c.commit(ops) // Start watching checks. Done after service registrations are built // since an error building them could leak watches. for _, service := range newIDs { - serviceID := makeTaskServiceID(allocID, newTask.Name, service) + serviceID := makeTaskServiceID(newTask.AllocID, newTask.Name, service, newTask.Canary) for _, check := range service.Checks { if check.TriggersRestarts() { checkID := makeCheckID(serviceID, check) - c.checkWatcher.Watch(allocID, newTask.Name, checkID, check, restarter) + c.checkWatcher.Watch(newTask.AllocID, newTask.Name, checkID, check, newTask.Restarter) } } } @@ -875,11 +883,11 @@ func (c *ServiceClient) UpdateTask(allocID string, existing, newTask *structs.Ta // RemoveTask from Consul. Removes all service entries and checks. // // Actual communication with Consul is done asynchronously (see Run). -func (c *ServiceClient) RemoveTask(allocID string, task *structs.Task) { +func (c *ServiceClient) RemoveTask(task *TaskServices) { ops := operations{} for _, service := range task.Services { - id := makeTaskServiceID(allocID, task.Name, service) + id := makeTaskServiceID(task.AllocID, task.Name, service, task.Canary) ops.deregServices = append(ops.deregServices, id) for _, check := range service.Checks { @@ -893,7 +901,7 @@ func (c *ServiceClient) RemoveTask(allocID string, task *structs.Task) { } // Remove the task from the alloc's registrations - c.removeTaskRegistration(allocID, task.Name) + c.removeTaskRegistration(task.AllocID, task.Name) // Now add them to the deregistration fields; main Run loop will update c.commit(&ops) @@ -1037,7 +1045,7 @@ func (c *ServiceClient) removeTaskRegistration(allocID, taskName string) { // Example Client ID: _nomad-client-ggnjpgl7yn7rgmvxzilmpvrzzvrszc7l // func makeAgentServiceID(role string, service *structs.Service) string { - return fmt.Sprintf("%s-%s-%s", nomadServicePrefix, role, service.Hash(role, "")) + return fmt.Sprintf("%s-%s-%s", nomadServicePrefix, role, service.Hash(role, "", false)) } // makeTaskServiceID creates a unique ID for identifying a task service in @@ -1045,8 +1053,8 @@ func makeAgentServiceID(role string, service *structs.Service) string { // Checks. This allows updates to merely compare IDs. // // Example Service ID: _nomad-task-TNM333JKJPM5AK4FAS3VXQLXFDWOF4VH -func makeTaskServiceID(allocID, taskName string, service *structs.Service) string { - return nomadTaskPrefix + service.Hash(allocID, taskName) +func makeTaskServiceID(allocID, taskName string, service *structs.Service, canary bool) string { + return nomadTaskPrefix + service.Hash(allocID, taskName, canary) } // makeCheckID creates a unique ID for a check. diff --git a/command/agent/consul/structs.go b/command/agent/consul/structs.go new file mode 100644 index 000000000000..2b61448ecf58 --- /dev/null +++ b/command/agent/consul/structs.go @@ -0,0 +1,67 @@ +package consul + +import ( + "github.com/hashicorp/nomad/client/driver" + cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/nomad/structs" +) + +type TaskServices struct { + AllocID string + + // Name of the task + Name string + + // Canary indicates whether or not the allocation is a canary + Canary bool + + // Restarter allows restarting the task depending on the task's + // check_restart stanzas. + Restarter TaskRestarter + + // Services and checks to register for the task. + Services []*structs.Service + + // Networks from the task's resources stanza. + Networks structs.Networks + + // DriverExec is the script executor for the task's driver. + DriverExec driver.ScriptExecutor + + // DriverNetwork is the network specified by the driver and may be nil. + DriverNetwork *cstructs.DriverNetwork +} + +func NewTaskServices(alloc *structs.Allocation, task *structs.Task, restarter TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) *TaskServices { + ts := TaskServices{ + AllocID: alloc.ID, + Name: task.Name, + Restarter: restarter, + Services: task.Services, + DriverExec: exec, + DriverNetwork: net, + } + + if task.Resources != nil { + ts.Networks = task.Resources.Networks + } + + if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Canary { + ts.Canary = true + } + + return &ts +} + +// Copy method for easing tests +func (t *TaskServices) Copy() *TaskServices { + newTS := new(TaskServices) + *newTS = *t + + // Deep copy Services + newTS.Services = make([]*structs.Service, len(t.Services)) + for i := range t.Services { + newTS.Services[i] = t.Services[i].Copy() + } + return newTS +} diff --git a/command/agent/consul/unit_test.go b/command/agent/consul/unit_test.go index 37157fcdcdd1..9b6ed2f785db 100644 --- a/command/agent/consul/unit_test.go +++ b/command/agent/consul/unit_test.go @@ -11,7 +11,9 @@ import ( "github.com/hashicorp/consul/api" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" "github.com/kr/pretty" @@ -25,19 +27,11 @@ const ( yPort = 1235 ) -func testTask() *structs.Task { - return &structs.Task{ - Name: "taskname", - Resources: &structs.Resources{ - Networks: []*structs.NetworkResource{ - { - DynamicPorts: []structs.Port{ - {Label: "x", Value: xPort}, - {Label: "y", Value: yPort}, - }, - }, - }, - }, +func testTask() *TaskServices { + return &TaskServices{ + AllocID: uuid.Generate(), + Name: "taskname", + Restarter: &restartRecorder{}, Services: []*structs.Service{ { Name: "taskname-service", @@ -45,27 +39,21 @@ func testTask() *structs.Task { Tags: []string{"tag1", "tag2"}, }, }, + Networks: []*structs.NetworkResource{ + { + DynamicPorts: []structs.Port{ + {Label: "x", Value: xPort}, + {Label: "y", Value: yPort}, + }, + }, + }, + DriverExec: newMockExec(), } } -// restartRecorder is a minimal TaskRestarter implementation that simply -// counts how many restarts were triggered. -type restartRecorder struct { - restarts int64 -} - -func (r *restartRecorder) Restart(source, reason string, failure bool) { - atomic.AddInt64(&r.restarts, 1) -} - -// testFakeCtx contains a fake Consul AgentAPI and implements the Exec -// interface to allow testing without running Consul. -type testFakeCtx struct { - ServiceClient *ServiceClient - FakeConsul *MockAgent - Task *structs.Task - Restarter *restartRecorder - +// mockExec implements the ScriptExecutor interface and will use an alternate +// implementation t.ExecFunc if non-nil. +type mockExec struct { // Ticked whenever a script is called execs chan int @@ -73,18 +61,40 @@ type testFakeCtx struct { ExecFunc func(ctx context.Context, cmd string, args []string) ([]byte, int, error) } -// Exec implements the ScriptExecutor interface and will use an alternate -// implementation t.ExecFunc if non-nil. -func (t *testFakeCtx) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) { +func newMockExec() *mockExec { + return &mockExec{ + execs: make(chan int, 100), + } +} + +func (m *mockExec) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) { select { - case t.execs <- 1: + case m.execs <- 1: default: } - if t.ExecFunc == nil { + if m.ExecFunc == nil { // Default impl is just "ok" return []byte("ok"), 0, nil } - return t.ExecFunc(ctx, cmd, args) + return m.ExecFunc(ctx, cmd, args) +} + +// restartRecorder is a minimal TaskRestarter implementation that simply +// counts how many restarts were triggered. +type restartRecorder struct { + restarts int64 +} + +func (r *restartRecorder) Restart(source, reason string, failure bool) { + atomic.AddInt64(&r.restarts, 1) +} + +// testFakeCtx contains a fake Consul AgentAPI +type testFakeCtx struct { + ServiceClient *ServiceClient + FakeConsul *MockAgent + Task *TaskServices + MockExec *mockExec } var errNoOps = fmt.Errorf("testing error: no pending operations") @@ -105,20 +115,19 @@ func (t *testFakeCtx) syncOnce() error { // A test Task is also provided. func setupFake(t *testing.T) *testFakeCtx { fc := NewMockAgent() + tt := testTask() return &testFakeCtx{ ServiceClient: NewServiceClient(fc, testlog.Logger(t)), FakeConsul: fc, - Task: testTask(), - Restarter: &restartRecorder{}, - execs: make(chan int, 100), + Task: tt, + MockExec: tt.DriverExec.(*mockExec), } } func TestConsul_ChangeTags(t *testing.T) { ctx := setupFake(t) - allocID := "allocid" - if err := ctx.ServiceClient.RegisterTask(allocID, ctx.Task, ctx.Restarter, nil, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -132,7 +141,7 @@ func TestConsul_ChangeTags(t *testing.T) { // Query the allocs registrations and then again when we update. The IDs // should change - reg1, err := ctx.ServiceClient.AllocRegistrations(allocID) + reg1, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID) if err != nil { t.Fatalf("Looking up alloc registration failed: %v", err) } @@ -157,10 +166,9 @@ func TestConsul_ChangeTags(t *testing.T) { } } - origTask := ctx.Task - ctx.Task = testTask() + origTask := ctx.Task.Copy() ctx.Task.Services[0].Tags[0] = "newtag" - if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, nil, nil); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } if err := ctx.syncOnce(); err != nil { @@ -184,7 +192,7 @@ func TestConsul_ChangeTags(t *testing.T) { } // Check again and ensure the IDs changed - reg2, err := ctx.ServiceClient.AllocRegistrations(allocID) + reg2, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID) if err != nil { t.Fatalf("Looking up alloc registration failed: %v", err) } @@ -242,7 +250,7 @@ func TestConsul_ChangePorts(t *testing.T) { }, } - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -285,8 +293,8 @@ func TestConsul_ChangePorts(t *testing.T) { case "c2": origScriptKey = k select { - case <-ctx.execs: - if n := len(ctx.execs); n > 0 { + case <-ctx.MockExec.execs: + if n := len(ctx.MockExec.execs); n > 0 { t.Errorf("expected 1 exec but found: %d", n+1) } case <-time.After(3 * time.Second): @@ -303,8 +311,7 @@ func TestConsul_ChangePorts(t *testing.T) { } // Now update the PortLabel on the Service and Check c3 - origTask := ctx.Task - ctx.Task = testTask() + origTask := ctx.Task.Copy() ctx.Task.Services[0].PortLabel = "y" ctx.Task.Services[0].Checks = []*structs.ServiceCheck{ { @@ -330,7 +337,7 @@ func TestConsul_ChangePorts(t *testing.T) { // Removed PortLabel; should default to service's (y) }, } - if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } if err := ctx.syncOnce(); err != nil { @@ -374,8 +381,8 @@ func TestConsul_ChangePorts(t *testing.T) { t.Errorf("expected key change for %s from %q", v.Name, origScriptKey) } select { - case <-ctx.execs: - if n := len(ctx.execs); n > 0 { + case <-ctx.MockExec.execs: + if n := len(ctx.MockExec.execs); n > 0 { t.Errorf("expected 1 exec but found: %d", n+1) } case <-time.After(3 * time.Second): @@ -411,8 +418,7 @@ func TestConsul_ChangeChecks(t *testing.T) { }, } - allocID := "allocid" - if err := ctx.ServiceClient.RegisterTask(allocID, ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -433,7 +439,7 @@ func TestConsul_ChangeChecks(t *testing.T) { // Query the allocs registrations and then again when we update. The IDs // should change - reg1, err := ctx.ServiceClient.AllocRegistrations(allocID) + reg1, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID) if err != nil { t.Fatalf("Looking up alloc registration failed: %v", err) } @@ -489,7 +495,7 @@ func TestConsul_ChangeChecks(t *testing.T) { PortLabel: "x", }, } - if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -547,7 +553,7 @@ func TestConsul_ChangeChecks(t *testing.T) { } // Check again and ensure the IDs changed - reg2, err := ctx.ServiceClient.AllocRegistrations(allocID) + reg2, err := ctx.ServiceClient.AllocRegistrations(ctx.Task.AllocID) if err != nil { t.Fatalf("Looking up alloc registration failed: %v", err) } @@ -603,7 +609,7 @@ func TestConsul_ChangeChecks(t *testing.T) { PortLabel: "x", }, } - if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil, ctx, nil); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } if err := ctx.syncOnce(); err != nil { @@ -646,7 +652,7 @@ func TestConsul_RegServices(t *testing.T) { }, } - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, nil, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -677,7 +683,7 @@ func TestConsul_RegServices(t *testing.T) { // Assert the check update is properly formed checkUpd := <-ctx.ServiceClient.checkWatcher.checkUpdateCh - if checkUpd.checkRestart.allocID != "allocid" { + if checkUpd.checkRestart.allocID != ctx.Task.AllocID { t.Fatalf("expected check's allocid to be %q but found %q", "allocid", checkUpd.checkRestart.allocID) } if expected := 200 * time.Millisecond; checkUpd.checkRestart.timeLimit != expected { @@ -687,7 +693,7 @@ func TestConsul_RegServices(t *testing.T) { // Make a change which will register a new service ctx.Task.Services[0].Name = "taskname-service2" ctx.Task.Services[0].Tags[0] = "tag3" - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, nil, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -737,7 +743,7 @@ func TestConsul_RegServices(t *testing.T) { } // Remove the new task - ctx.ServiceClient.RemoveTask("allocid", ctx.Task) + ctx.ServiceClient.RemoveTask(ctx.Task) if err := ctx.syncOnce(); err != nil { t.Fatalf("unexpected error syncing task: %v", err) } @@ -787,7 +793,7 @@ func TestConsul_ShutdownOK(t *testing.T) { go ctx.ServiceClient.Run() // Register a task and agent - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -849,7 +855,7 @@ func TestConsul_ShutdownSlow(t *testing.T) { // Make Exec slow, but not too slow waiter := make(chan struct{}) - ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) { + ctx.MockExec.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) { select { case <-waiter: default: @@ -865,7 +871,7 @@ func TestConsul_ShutdownSlow(t *testing.T) { go ctx.ServiceClient.Run() // Register a task and agent - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -924,7 +930,7 @@ func TestConsul_ShutdownBlocked(t *testing.T) { // Make Exec block forever waiter := make(chan struct{}) - ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) { + ctx.MockExec.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) { close(waiter) <-block return []byte{}, 0, nil @@ -936,7 +942,7 @@ func TestConsul_ShutdownBlocked(t *testing.T) { go ctx.ServiceClient.Run() // Register a task and agent - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -988,7 +994,7 @@ func TestConsul_CancelScript(t *testing.T) { }, } - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -1007,7 +1013,7 @@ func TestConsul_CancelScript(t *testing.T) { for i := 0; i < 2; i++ { select { - case <-ctx.execs: + case <-ctx.MockExec.execs: // Script ran as expected! case <-time.After(3 * time.Second): t.Fatalf("timed out waiting for script check to run") @@ -1025,7 +1031,7 @@ func TestConsul_CancelScript(t *testing.T) { }, } - if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, ctx.Restarter, ctx, nil); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -1044,7 +1050,7 @@ func TestConsul_CancelScript(t *testing.T) { // Make sure exec wasn't called again select { - case <-ctx.execs: + case <-ctx.MockExec.execs: t.Errorf("unexpected execution of script; was goroutine not cancelled?") case <-time.After(100 * time.Millisecond): // No unexpected script execs @@ -1104,7 +1110,7 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) { }, } - net := &cstructs.DriverNetwork{ + ctx.Task.DriverNetwork = &cstructs.DriverNetwork{ PortMap: map[string]int{ "x": 8888, "y": 9999, @@ -1113,7 +1119,7 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) { AutoAdvertise: true, } - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -1129,9 +1135,9 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) { switch v.Name { case ctx.Task.Services[0].Name: // x // Since DriverNetwork.AutoAdvertise=true, driver ports should be used - if v.Port != net.PortMap["x"] { + if v.Port != ctx.Task.DriverNetwork.PortMap["x"] { t.Errorf("expected service %s's port to be %d but found %d", - v.Name, net.PortMap["x"], v.Port) + v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port) } // The order of checks in Consul is not guaranteed to // be the same as their order in the Task definition, @@ -1159,13 +1165,13 @@ func TestConsul_DriverNetwork_AutoUse(t *testing.T) { } case ctx.Task.Services[1].Name: // y // Service should be container ip:port - if v.Address != net.IP { + if v.Address != ctx.Task.DriverNetwork.IP { t.Errorf("expected service %s's address to be %s but found %s", - v.Name, net.IP, v.Address) + v.Name, ctx.Task.DriverNetwork.IP, v.Address) } - if v.Port != net.PortMap["y"] { + if v.Port != ctx.Task.DriverNetwork.PortMap["y"] { t.Errorf("expected service %s's port to be %d but found %d", - v.Name, net.PortMap["x"], v.Port) + v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port) } // Check should be host ip:port if v.Checks[0].TCP != ":1235" { // yPort @@ -1208,7 +1214,7 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) { }, } - net := &cstructs.DriverNetwork{ + ctx.Task.DriverNetwork = &cstructs.DriverNetwork{ PortMap: map[string]int{ "x": 8888, "y": 9999, @@ -1217,7 +1223,7 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) { AutoAdvertise: false, } - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } @@ -1239,13 +1245,13 @@ func TestConsul_DriverNetwork_NoAutoUse(t *testing.T) { } case ctx.Task.Services[1].Name: // y + driver mode // Service should be container ip:port - if v.Address != net.IP { + if v.Address != ctx.Task.DriverNetwork.IP { t.Errorf("expected service %s's address to be %s but found %s", - v.Name, net.IP, v.Address) + v.Name, ctx.Task.DriverNetwork.IP, v.Address) } - if v.Port != net.PortMap["y"] { + if v.Port != ctx.Task.DriverNetwork.PortMap["y"] { t.Errorf("expected service %s's port to be %d but found %d", - v.Name, net.PortMap["x"], v.Port) + v.Name, ctx.Task.DriverNetwork.PortMap["x"], v.Port) } case ctx.Task.Services[2].Name: // y + host mode if v.Port != yPort { @@ -1272,7 +1278,7 @@ func TestConsul_DriverNetwork_Change(t *testing.T) { }, } - net := &cstructs.DriverNetwork{ + ctx.Task.DriverNetwork = &cstructs.DriverNetwork{ PortMap: map[string]int{ "x": 8888, "y": 9999, @@ -1304,31 +1310,63 @@ func TestConsul_DriverNetwork_Change(t *testing.T) { } // Initial service should advertise host port x - if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx.Restarter, ctx, net); err != nil { + if err := ctx.ServiceClient.RegisterTask(ctx.Task); err != nil { t.Fatalf("unexpected error registering task: %v", err) } syncAndAssertPort(xPort) // UpdateTask to use Host (shouldn't change anything) - orig := ctx.Task.Copy() + origTask := ctx.Task.Copy() ctx.Task.Services[0].AddressMode = structs.AddressModeHost - if err := ctx.ServiceClient.UpdateTask("allocid", orig, ctx.Task, ctx.Restarter, ctx, net); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error updating task: %v", err) } syncAndAssertPort(xPort) // UpdateTask to use Driver (*should* change IP and port) - orig = ctx.Task.Copy() + origTask = ctx.Task.Copy() ctx.Task.Services[0].AddressMode = structs.AddressModeDriver - if err := ctx.ServiceClient.UpdateTask("allocid", orig, ctx.Task, ctx.Restarter, ctx, net); err != nil { + if err := ctx.ServiceClient.UpdateTask(origTask, ctx.Task); err != nil { t.Fatalf("unexpected error updating task: %v", err) } - syncAndAssertPort(net.PortMap["x"]) + syncAndAssertPort(ctx.Task.DriverNetwork.PortMap["x"]) +} + +// TestConsul_CanaryTags asserts CanaryTags are used when Canary=true +func TestConsul_CanaryTags(t *testing.T) { + t.Parallel() + require := require.New(t) + ctx := setupFake(t) + + canaryTags := []string{"tag1", "canary"} + ctx.Task.Canary = true + ctx.Task.Services[0].CanaryTags = canaryTags + + require.NoError(ctx.ServiceClient.RegisterTask(ctx.Task)) + require.NoError(ctx.syncOnce()) + require.Len(ctx.FakeConsul.services, 1) + for _, service := range ctx.FakeConsul.services { + require.Equal(canaryTags, service.Tags) + } + + // Disable canary and assert tags are not the canary tags + origTask := ctx.Task.Copy() + ctx.Task.Canary = false + require.NoError(ctx.ServiceClient.UpdateTask(origTask, ctx.Task)) + require.NoError(ctx.syncOnce()) + require.Len(ctx.FakeConsul.services, 1) + for _, service := range ctx.FakeConsul.services { + require.NotEqual(canaryTags, service.Tags) + } + + ctx.ServiceClient.RemoveTask(ctx.Task) + require.NoError(ctx.syncOnce()) + require.Len(ctx.FakeConsul.services, 0) } // TestConsul_PeriodicSync asserts that Nomad periodically reconciles with diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 510680002d24..9ff26acc66dd 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -693,13 +693,14 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) { if taskGroup.Update != nil { tg.Update = &structs.UpdateStrategy{ - Stagger: *taskGroup.Update.Stagger, - MaxParallel: *taskGroup.Update.MaxParallel, - HealthCheck: *taskGroup.Update.HealthCheck, - MinHealthyTime: *taskGroup.Update.MinHealthyTime, - HealthyDeadline: *taskGroup.Update.HealthyDeadline, - AutoRevert: *taskGroup.Update.AutoRevert, - Canary: *taskGroup.Update.Canary, + Stagger: *taskGroup.Update.Stagger, + MaxParallel: *taskGroup.Update.MaxParallel, + HealthCheck: *taskGroup.Update.HealthCheck, + MinHealthyTime: *taskGroup.Update.MinHealthyTime, + HealthyDeadline: *taskGroup.Update.HealthyDeadline, + ProgressDeadline: *taskGroup.Update.ProgressDeadline, + AutoRevert: *taskGroup.Update.AutoRevert, + Canary: *taskGroup.Update.Canary, } } @@ -743,6 +744,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) { Name: service.Name, PortLabel: service.PortLabel, Tags: service.Tags, + CanaryTags: service.CanaryTags, AddressMode: service.AddressMode, } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index fb129dc918b0..d92433f4ed73 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -1161,13 +1161,14 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, Update: &api.UpdateStrategy{ - Stagger: helper.TimeToPtr(1 * time.Second), - MaxParallel: helper.IntToPtr(5), - HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), - MinHealthyTime: helper.TimeToPtr(1 * time.Minute), - HealthyDeadline: helper.TimeToPtr(3 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(1), + Stagger: helper.TimeToPtr(1 * time.Second), + MaxParallel: helper.IntToPtr(5), + HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), + MinHealthyTime: helper.TimeToPtr(1 * time.Minute), + HealthyDeadline: helper.TimeToPtr(3 * time.Minute), + ProgressDeadline: helper.TimeToPtr(3 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(1), }, Periodic: &api.PeriodicConfig{ Enabled: helper.BoolToPtr(true), @@ -1222,10 +1223,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Migrate: helper.BoolToPtr(true), }, Update: &api.UpdateStrategy{ - HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks), - MinHealthyTime: helper.TimeToPtr(2 * time.Minute), - HealthyDeadline: helper.TimeToPtr(5 * time.Minute), - AutoRevert: helper.BoolToPtr(true), + HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks), + MinHealthyTime: helper.TimeToPtr(2 * time.Minute), + HealthyDeadline: helper.TimeToPtr(5 * time.Minute), + ProgressDeadline: helper.TimeToPtr(5 * time.Minute), + AutoRevert: helper.BoolToPtr(true), }, Meta: map[string]string{ @@ -1253,10 +1255,11 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Services: []*api.Service{ { - Id: "id", - Name: "serviceA", - Tags: []string{"1", "2"}, - PortLabel: "foo", + Id: "id", + Name: "serviceA", + Tags: []string{"1", "2"}, + CanaryTags: []string{"3", "4"}, + PortLabel: "foo", CheckRestart: &api.CheckRestart{ Limit: 4, Grace: helper.TimeToPtr(11 * time.Second), @@ -1446,13 +1449,14 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Migrate: true, }, Update: &structs.UpdateStrategy{ - Stagger: 1 * time.Second, - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 2 * time.Minute, - HealthyDeadline: 5 * time.Minute, - AutoRevert: true, - Canary: 1, + Stagger: 1 * time.Second, + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 2 * time.Minute, + HealthyDeadline: 5 * time.Minute, + ProgressDeadline: 5 * time.Minute, + AutoRevert: true, + Canary: 1, }, Meta: map[string]string{ "key": "value", @@ -1480,6 +1484,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { { Name: "serviceA", Tags: []string{"1", "2"}, + CanaryTags: []string{"3", "4"}, PortLabel: "foo", AddressMode: "auto", Checks: []*structs.ServiceCheck{ diff --git a/command/alloc_status.go b/command/alloc_status.go index b3e41fa6ae11..677200da98da 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -246,34 +246,22 @@ func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength if alloc.DeploymentID != "" { health := "unset" - if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil { - if *alloc.DeploymentStatus.Healthy { - health = "healthy" - } else { - health = "unhealthy" + canary := false + if alloc.DeploymentStatus != nil { + if alloc.DeploymentStatus.Healthy != nil { + if *alloc.DeploymentStatus.Healthy { + health = "healthy" + } else { + health = "unhealthy" + } } + + canary = alloc.DeploymentStatus.Canary } basic = append(basic, fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)), fmt.Sprintf("Deployment Health|%s", health)) - - // Check if this allocation is a canary - deployment, _, err := client.Deployments().Info(alloc.DeploymentID, nil) - if err != nil { - return "", fmt.Errorf("Error querying deployment %q: %s", alloc.DeploymentID, err) - } - - canary := false - if state, ok := deployment.TaskGroups[alloc.TaskGroup]; ok { - for _, id := range state.PlacedCanaries { - if id == alloc.ID { - canary = true - break - } - } - } - if canary { basic = append(basic, fmt.Sprintf("Canary|%v", true)) } diff --git a/command/deployment_status.go b/command/deployment_status.go index 5380ea3107f8..c5257f239b10 100644 --- a/command/deployment_status.go +++ b/command/deployment_status.go @@ -195,7 +195,7 @@ func formatDeployment(d *api.Deployment, uuidLength int) string { func formatDeploymentGroups(d *api.Deployment, uuidLength int) string { // Detect if we need to add these columns - canaries, autorevert := false, false + var canaries, autorevert, progressDeadline bool tgNames := make([]string, 0, len(d.TaskGroups)) for name, state := range d.TaskGroups { tgNames = append(tgNames, name) @@ -205,6 +205,9 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string { if state.DesiredCanaries > 0 { canaries = true } + if state.ProgressDeadline != 0 { + progressDeadline = true + } } // Sort the task group names to get a reliable ordering @@ -223,6 +226,9 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string { rowString += "Canaries|" } rowString += "Placed|Healthy|Unhealthy" + if progressDeadline { + rowString += "|Progress Deadline" + } rows := make([]string, len(d.TaskGroups)+1) rows[0] = rowString @@ -245,6 +251,13 @@ func formatDeploymentGroups(d *api.Deployment, uuidLength int) string { row += fmt.Sprintf("%d|", state.DesiredCanaries) } row += fmt.Sprintf("%d|%d|%d", state.PlacedAllocs, state.HealthyAllocs, state.UnhealthyAllocs) + if progressDeadline { + if state.RequireProgressBy.IsZero() { + row += fmt.Sprintf("|%v", "N/A") + } else { + row += fmt.Sprintf("|%v", formatTime(state.RequireProgressBy)) + } + } rows[i] = row i++ } diff --git a/e2e/consul/canary_tags_test.go b/e2e/consul/canary_tags_test.go new file mode 100644 index 000000000000..3b6e953c52e6 --- /dev/null +++ b/e2e/consul/canary_tags_test.go @@ -0,0 +1,161 @@ +package consul_test + +import ( + "flag" + "testing" + "time" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/jobspec" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var integration = flag.Bool("integration", false, "run integration tests") + +func TestConsul(t *testing.T) { + if !*integration { + t.Skip("skipping test in non-integration mode.") + } + RegisterFailHandler(Fail) + RunSpecs(t, "Consul Canary Tags Test") +} + +var _ = Describe("Consul Canary Tags Test", func() { + + var ( + agent *consulapi.Agent + allocations *api.Allocations + deployments *api.Deployments + jobs *api.Jobs + system *api.System + job *api.Job + specFile string + ) + + BeforeSuite(func() { + consulConf := consulapi.DefaultConfig() + consulClient, err := consulapi.NewClient(consulConf) + Expect(err).ShouldNot(HaveOccurred()) + agent = consulClient.Agent() + + conf := api.DefaultConfig() + client, err := api.NewClient(conf) + Expect(err).ShouldNot(HaveOccurred()) + allocations = client.Allocations() + deployments = client.Deployments() + jobs = client.Jobs() + system = client.System() + }) + + JustBeforeEach(func() { + var err error + job, err = jobspec.ParseFile(specFile) + Expect(err).ShouldNot(HaveOccurred()) + job.ID = helper.StringToPtr(*job.ID + uuid.Generate()[22:]) + resp, _, err := jobs.Register(job, nil) + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.EvalID).ShouldNot(BeEmpty()) + }) + + AfterEach(func() { + jobs.Deregister(*job.ID, true, nil) + system.GarbageCollect() + }) + + Describe("Consul Canary Tags Test", func() { + Context("Canary Tags", func() { + BeforeEach(func() { + specFile = "input/canary_tags.hcl" + }) + + It("Should set and unset canary tags", func() { + + // Eventually be running and healthy + Eventually(func() []string { + deploys, _, err := jobs.Deployments(*job.ID, nil) + Expect(err).ShouldNot(HaveOccurred()) + healthyDeploys := make([]string, 0, len(deploys)) + for _, d := range deploys { + if d.Status == "successful" { + healthyDeploys = append(healthyDeploys, d.ID) + } + } + return healthyDeploys + }, 5*time.Second, 20*time.Millisecond).Should(HaveLen(1)) + + // Start a deployment + job.Meta = map[string]string{"version": "2"} + resp, _, err := jobs.Register(job, nil) + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.EvalID).ShouldNot(BeEmpty()) + + // Eventually have a canary + var deploys []*api.Deployment + Eventually(func() []*api.Deployment { + deploys, _, err = jobs.Deployments(*job.ID, nil) + Expect(err).ShouldNot(HaveOccurred()) + return deploys + }, 2*time.Second, 20*time.Millisecond).Should(HaveLen(2)) + + var deploy *api.Deployment + Eventually(func() []string { + deploy, _, err = deployments.Info(deploys[0].ID, nil) + Expect(err).ShouldNot(HaveOccurred()) + return deploy.TaskGroups["consul_canary_test"].PlacedCanaries + }, 2*time.Second, 20*time.Millisecond).Should(HaveLen(1)) + + Eventually(func() bool { + allocID := deploy.TaskGroups["consul_canary_test"].PlacedCanaries[0] + alloc, _, err := allocations.Info(allocID, nil) + Expect(err).ShouldNot(HaveOccurred()) + return alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil && *alloc.DeploymentStatus.Healthy + }, 3*time.Second, 20*time.Millisecond).Should(BeTrue()) + + // Check Consul for canary tags + Eventually(func() []string { + services, err := agent.Services() + Expect(err).ShouldNot(HaveOccurred()) + for _, v := range services { + if v.Service == "canarytest" { + return v.Tags + } + } + return nil + }, 2*time.Second, 20*time.Millisecond).Should( + Equal([]string{"foo", "canary"})) + + // Manually promote + { + resp, _, err := deployments.PromoteAll(deploys[0].ID, nil) + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.EvalID).ShouldNot(BeEmpty()) + } + + // Eventually canary is removed + Eventually(func() bool { + allocID := deploy.TaskGroups["consul_canary_test"].PlacedCanaries[0] + alloc, _, err := allocations.Info(allocID, nil) + Expect(err).ShouldNot(HaveOccurred()) + return alloc.DeploymentStatus.Canary + }, 2*time.Second, 20*time.Millisecond).Should(BeFalse()) + + // Check Consul canary tags were removed + Eventually(func() []string { + services, err := agent.Services() + Expect(err).ShouldNot(HaveOccurred()) + for _, v := range services { + if v.Service == "canarytest" { + return v.Tags + } + } + return nil + }, 2*time.Second, 20*time.Millisecond).Should( + Equal([]string{"foo", "bar"})) + }) + }) + }) +}) diff --git a/e2e/consul/input/canary_tags.hcl b/e2e/consul/input/canary_tags.hcl new file mode 100644 index 000000000000..da98db490879 --- /dev/null +++ b/e2e/consul/input/canary_tags.hcl @@ -0,0 +1,36 @@ +job "consul_canary_test" { + datacenters = ["dc1"] + + group "consul_canary_test" { + count = 2 + + task "consul_canary_test" { + driver = "mock_driver" + + config { + run_for = "10m" + exit_code = 9 + } + + service { + name = "canarytest" + tags = ["foo", "bar"] + canary_tags = ["foo", "canary"] + } + } + + update { + max_parallel = 1 + canary = 1 + min_healthy_time = "1s" + health_check = "task_states" + auto_revert = false + } + + restart { + attempts = 0 + delay = "0s" + mode = "fail" + } + } +} diff --git a/e2e/rescheduling/input/rescheduling_canary_autorevert.hcl b/e2e/rescheduling/input/rescheduling_canary_autorevert.hcl index cef2e5a37072..b1194cbf949a 100644 --- a/e2e/rescheduling/input/rescheduling_canary_autorevert.hcl +++ b/e2e/rescheduling/input/rescheduling_canary_autorevert.hcl @@ -18,8 +18,9 @@ job "test" { canary = 3 max_parallel = 1 min_healthy_time = "1s" - healthy_deadline = "1m" auto_revert = true + healthy_deadline = "2s" + progress_deadline = "3s" } restart { diff --git a/e2e/rescheduling/input/rescheduling_maxp.hcl b/e2e/rescheduling/input/rescheduling_maxp.hcl index 331335643bf0..7a5f30da188c 100644 --- a/e2e/rescheduling/input/rescheduling_maxp.hcl +++ b/e2e/rescheduling/input/rescheduling_maxp.hcl @@ -17,8 +17,9 @@ job "demo2" { update { max_parallel = 1 min_healthy_time = "1s" - healthy_deadline = "1m" auto_revert = false + healthy_deadline = "2s" + progress_deadline = "5s" } restart { diff --git a/e2e/rescheduling/input/rescheduling_maxp_autorevert.hcl b/e2e/rescheduling/input/rescheduling_maxp_autorevert.hcl index d9ae0021ae1f..72d0554265af 100644 --- a/e2e/rescheduling/input/rescheduling_maxp_autorevert.hcl +++ b/e2e/rescheduling/input/rescheduling_maxp_autorevert.hcl @@ -10,15 +10,16 @@ job "demo3" { config { command = "bash" - args = ["-c", "sleep 5000"] + args = ["-c", "sleep 15000"] } } update { max_parallel = 1 min_healthy_time = "1s" - healthy_deadline = "1m" auto_revert = true + healthy_deadline = "2s" + progress_deadline = "3s" } restart { diff --git a/e2e/rescheduling/input/rescheduling_update.hcl b/e2e/rescheduling/input/rescheduling_update.hcl index d4ecd64810a6..6906f9f0b96e 100644 --- a/e2e/rescheduling/input/rescheduling_update.hcl +++ b/e2e/rescheduling/input/rescheduling_update.hcl @@ -16,8 +16,10 @@ job "test4" { update { max_parallel = 1 - min_healthy_time = "10s" + min_healthy_time = "3s" auto_revert = false + healthy_deadline = "5s" + progress_deadline = "10s" } restart { diff --git a/e2e/rescheduling/server_side_restarts_test.go b/e2e/rescheduling/server_side_restarts_test.go index 9f0f402df4af..7711a8a5783d 100644 --- a/e2e/rescheduling/server_side_restarts_test.go +++ b/e2e/rescheduling/server_side_restarts_test.go @@ -168,11 +168,11 @@ var _ = Describe("Server Side Restart Tests", func() { ConsistOf([]string{"running", "running", "running"})) }) Context("Updating job to make allocs fail", func() { - It("Should have no rescheduled allocs", func() { + It("Should have rescheduled allocs until progress deadline", func() { job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"} _, _, err := jobs.Register(job, nil) Expect(err).ShouldNot(HaveOccurred()) - Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty()) + Eventually(allocStatusesRescheduled, 5*time.Second, time.Second).ShouldNot(BeEmpty()) }) }) @@ -192,22 +192,23 @@ var _ = Describe("Server Side Restart Tests", func() { }) Context("Updating job to make allocs fail", func() { - It("Should have no rescheduled allocs", func() { + It("Should have rescheduled allocs until progress deadline", func() { job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"} _, _, err := jobs.Register(job, nil) Expect(err).ShouldNot(HaveOccurred()) - Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty()) + Eventually(allocStatusesRescheduled, 5*time.Second, time.Second).ShouldNot(BeEmpty()) // Verify new deployment and its status + // Deployment status should be running (because of progress deadline) time.Sleep(3 * time.Second) //TODO(preetha) figure out why this wasn't working with ginkgo constructs Eventually(deploymentStatus(), 2*time.Second, time.Second).Should( - ContainElement(structs.DeploymentStatusFailed)) + ContainElement(structs.DeploymentStatusRunning)) }) }) }) - Context("Reschedule with canary and auto revert ", func() { + Context("Reschedule with canary, auto revert with short progress deadline ", func() { BeforeEach(func() { specFile = "input/rescheduling_canary_autorevert.hcl" }) @@ -228,11 +229,10 @@ var _ = Describe("Server Side Restart Tests", func() { // Wait for the revert Eventually(allocStatuses, 3*time.Second, time.Second).Should( ConsistOf([]string{"failed", "failed", "failed", "running", "running", "running"})) - // Verify new deployment and its status // There should be one successful, one failed, and one more successful (after revert) time.Sleep(5 * time.Second) //TODO(preetha) figure out why this wasn't working with ginkgo constructs - Eventually(deploymentStatus(), 2*time.Second, time.Second).Should( + Eventually(deploymentStatus(), 5*time.Second, time.Second).Should( ConsistOf(structs.DeploymentStatusSuccessful, structs.DeploymentStatusFailed, structs.DeploymentStatusSuccessful)) }) @@ -252,11 +252,11 @@ var _ = Describe("Server Side Restart Tests", func() { }) Context("Updating job to make allocs fail", func() { - It("Should have no rescheduled allocs", func() { + It("Should have rescheduled allocs till progress deadline", func() { job.TaskGroups[0].Tasks[0].Config["args"] = []string{"-c", "lol"} _, _, err := jobs.Register(job, nil) Expect(err).ShouldNot(HaveOccurred()) - Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty()) + Eventually(allocStatusesRescheduled, 3*time.Second, time.Second).ShouldNot(BeEmpty()) // Should have 1 failed from max_parallel Eventually(allocStatuses, 3*time.Second, time.Second).Should( @@ -265,13 +265,13 @@ var _ = Describe("Server Side Restart Tests", func() { // Verify new deployment and its status time.Sleep(2 * time.Second) Eventually(deploymentStatus(), 2*time.Second, time.Second).Should( - ContainElement(structs.DeploymentStatusFailed)) + ContainElement(structs.DeploymentStatusRunning)) }) }) }) - Context("Reschedule with max parallel and auto revert true ", func() { + Context("Reschedule with max parallel, auto revert true and short progress deadline", func() { BeforeEach(func() { specFile = "input/rescheduling_maxp_autorevert.hcl" }) @@ -290,7 +290,7 @@ var _ = Describe("Server Side Restart Tests", func() { Eventually(allocStatusesRescheduled, 2*time.Second, time.Second).Should(BeEmpty()) // Wait for the revert - Eventually(allocStatuses, 3*time.Second, time.Second).Should( + Eventually(allocStatuses, 5*time.Second, time.Second).Should( ConsistOf([]string{"complete", "failed", "running", "running", "running"})) // Verify new deployment and its status diff --git a/jobspec/parse.go b/jobspec/parse.go index 63157140e2e5..143c5f02fa38 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -987,6 +987,7 @@ func parseServices(jobName string, taskGroupName string, task *api.Task, service valid := []string{ "name", "tags", + "canary_tags", "port", "check", "address_mode", @@ -1322,6 +1323,7 @@ func parseUpdate(result **api.UpdateStrategy, list *ast.ObjectList) error { "health_check", "min_healthy_time", "healthy_deadline", + "progress_deadline", "auto_revert", "canary", } diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 69875f7713df..f2839eb06b77 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -47,13 +47,14 @@ func TestParse(t *testing.T) { }, Update: &api.UpdateStrategy{ - Stagger: helper.TimeToPtr(60 * time.Second), - MaxParallel: helper.IntToPtr(2), - HealthCheck: helper.StringToPtr("manual"), - MinHealthyTime: helper.TimeToPtr(10 * time.Second), - HealthyDeadline: helper.TimeToPtr(10 * time.Minute), - AutoRevert: helper.BoolToPtr(true), - Canary: helper.IntToPtr(1), + Stagger: helper.TimeToPtr(60 * time.Second), + MaxParallel: helper.IntToPtr(2), + HealthCheck: helper.StringToPtr("manual"), + MinHealthyTime: helper.TimeToPtr(10 * time.Second), + HealthyDeadline: helper.TimeToPtr(10 * time.Minute), + ProgressDeadline: helper.TimeToPtr(10 * time.Minute), + AutoRevert: helper.BoolToPtr(true), + Canary: helper.IntToPtr(1), }, TaskGroups: []*api.TaskGroup{ @@ -103,12 +104,13 @@ func TestParse(t *testing.T) { SizeMB: helper.IntToPtr(150), }, Update: &api.UpdateStrategy{ - MaxParallel: helper.IntToPtr(3), - HealthCheck: helper.StringToPtr("checks"), - MinHealthyTime: helper.TimeToPtr(1 * time.Second), - HealthyDeadline: helper.TimeToPtr(1 * time.Minute), - AutoRevert: helper.BoolToPtr(false), - Canary: helper.IntToPtr(2), + MaxParallel: helper.IntToPtr(3), + HealthCheck: helper.StringToPtr("checks"), + MinHealthyTime: helper.TimeToPtr(1 * time.Second), + HealthyDeadline: helper.TimeToPtr(1 * time.Minute), + ProgressDeadline: helper.TimeToPtr(1 * time.Minute), + AutoRevert: helper.BoolToPtr(false), + Canary: helper.IntToPtr(2), }, Migrate: &api.MigrateStrategy{ MaxParallel: helper.IntToPtr(2), @@ -131,8 +133,9 @@ func TestParse(t *testing.T) { }, Services: []*api.Service{ { - Tags: []string{"foo", "bar"}, - PortLabel: "http", + Tags: []string{"foo", "bar"}, + CanaryTags: []string{"canary", "bam"}, + PortLabel: "http", Checks: []api.ServiceCheck{ { Name: "check-name", diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 11f40b883ad7..12c7d9965dfa 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -22,6 +22,7 @@ job "binstore-storagelocker" { health_check = "manual" min_healthy_time = "10s" healthy_deadline = "10m" + progress_deadline = "10m" auto_revert = true canary = 1 } @@ -63,6 +64,7 @@ job "binstore-storagelocker" { health_check = "checks" min_healthy_time = "1s" healthy_deadline = "1m" + progress_deadline = "1m" auto_revert = false canary = 2 } @@ -99,6 +101,7 @@ job "binstore-storagelocker" { service { tags = ["foo", "bar"] + canary_tags = ["canary", "bam"] port = "http" check { diff --git a/nomad/deployment_endpoint_test.go b/nomad/deployment_endpoint_test.go index 97e0ae416ed5..fa9ca5f860fb 100644 --- a/nomad/deployment_endpoint_test.go +++ b/nomad/deployment_endpoint_test.go @@ -472,9 +472,9 @@ func TestDeploymentEndpoint_Promote(t *testing.T) { j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 - j.TaskGroups[0].Update.Canary = 2 + j.TaskGroups[0].Update.Canary = 1 d := mock.Deployment() - d.TaskGroups["web"].DesiredCanaries = 2 + d.TaskGroups["web"].DesiredCanaries = 1 d.JobID = j.ID a := mock.Alloc() d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} @@ -536,9 +536,9 @@ func TestDeploymentEndpoint_Promote_ACL(t *testing.T) { j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 - j.TaskGroups[0].Update.Canary = 2 + j.TaskGroups[0].Update.Canary = 1 d := mock.Deployment() - d.TaskGroups["web"].DesiredCanaries = 2 + d.TaskGroups["web"].DesiredCanaries = 1 d.JobID = j.ID a := mock.Alloc() d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} diff --git a/nomad/deployment_watcher_shims.go b/nomad/deployment_watcher_shims.go index c703feb7fba8..7f2e34685cba 100644 --- a/nomad/deployment_watcher_shims.go +++ b/nomad/deployment_watcher_shims.go @@ -25,14 +25,6 @@ func (d *deploymentWatcherRaftShim) convertApplyErrors(applyResp interface{}, in return index, err } -func (d *deploymentWatcherRaftShim) UpsertEvals(evals []*structs.Evaluation) (uint64, error) { - update := &structs.EvalUpdateRequest{ - Evals: evals, - } - fsmErrIntf, index, raftErr := d.apply(structs.EvalUpdateRequestType, update) - return d.convertApplyErrors(fsmErrIntf, index, raftErr) -} - func (d *deploymentWatcherRaftShim) UpsertJob(job *structs.Job) (uint64, error) { job.SetSubmitTime() update := &structs.JobRegisterRequest{ @@ -56,3 +48,8 @@ func (d *deploymentWatcherRaftShim) UpdateDeploymentAllocHealth(req *structs.App fsmErrIntf, index, raftErr := d.apply(structs.DeploymentAllocHealthRequestType, req) return d.convertApplyErrors(fsmErrIntf, index, raftErr) } + +func (d *deploymentWatcherRaftShim) UpdateAllocDesiredTransition(req *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) { + fsmErrIntf, index, raftErr := d.apply(structs.AllocUpdateDesiredTransitionRequestType, req) + return d.convertApplyErrors(fsmErrIntf, index, raftErr) +} diff --git a/nomad/deploymentwatcher/batcher.go b/nomad/deploymentwatcher/batcher.go index 0d730a2cd0a6..3d0f34eb9570 100644 --- a/nomad/deploymentwatcher/batcher.go +++ b/nomad/deploymentwatcher/batcher.go @@ -7,58 +7,62 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) -// EvalBatcher is used to batch the creation of evaluations -type EvalBatcher struct { +// AllocUpdateBatcher is used to batch the updates to the desired transitions +// of allocations and the creation of evals. +type AllocUpdateBatcher struct { // batch is the batching duration batch time.Duration - // raft is used to actually commit the evaluations + // raft is used to actually commit the updates raft DeploymentRaftEndpoints // workCh is used to pass evaluations to the daemon process - workCh chan *evalWrapper + workCh chan *updateWrapper // ctx is used to exit the daemon batcher ctx context.Context } -// NewEvalBatcher returns an EvalBatcher that uses the passed raft endpoints to -// create the evaluations and exits the batcher when the passed exit channel is -// closed. -func NewEvalBatcher(batchDuration time.Duration, raft DeploymentRaftEndpoints, ctx context.Context) *EvalBatcher { - b := &EvalBatcher{ +// NewAllocUpdateBatcher returns an AllocUpdateBatcher that uses the passed raft endpoints to +// create the allocation desired transition updates and new evaluations and +// exits the batcher when the passed exit channel is closed. +func NewAllocUpdateBatcher(batchDuration time.Duration, raft DeploymentRaftEndpoints, ctx context.Context) *AllocUpdateBatcher { + b := &AllocUpdateBatcher{ batch: batchDuration, raft: raft, ctx: ctx, - workCh: make(chan *evalWrapper, 10), + workCh: make(chan *updateWrapper, 10), } go b.batcher() return b } -// CreateEval batches the creation of the evaluation and returns a future that -// tracks the evaluations creation. -func (b *EvalBatcher) CreateEval(e *structs.Evaluation) *EvalFuture { - wrapper := &evalWrapper{ - e: e, - f: make(chan *EvalFuture, 1), +// CreateUpdate batches the allocation desired transition update and returns a +// future that tracks the completion of the request. +func (b *AllocUpdateBatcher) CreateUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) *BatchFuture { + wrapper := &updateWrapper{ + allocs: allocs, + e: eval, + f: make(chan *BatchFuture, 1), } b.workCh <- wrapper return <-wrapper.f } -type evalWrapper struct { - e *structs.Evaluation - f chan *EvalFuture +type updateWrapper struct { + allocs map[string]*structs.DesiredTransition + e *structs.Evaluation + f chan *BatchFuture } // batcher is the long lived batcher goroutine -func (b *EvalBatcher) batcher() { +func (b *AllocUpdateBatcher) batcher() { var timerCh <-chan time.Time + allocs := make(map[string]*structs.DesiredTransition) evals := make(map[string]*structs.Evaluation) - future := NewEvalFuture() + future := NewBatchFuture() for { select { case <-b.ctx.Done(): @@ -68,59 +72,68 @@ func (b *EvalBatcher) batcher() { timerCh = time.After(b.batch) } - // Store the eval and attach the future + // Store the eval and alloc updates, and attach the future evals[w.e.DeploymentID] = w.e + for id, upd := range w.allocs { + allocs[id] = upd + } + w.f <- future case <-timerCh: // Capture the future and create a new one f := future - future = NewEvalFuture() + future = NewBatchFuture() // Shouldn't be possible if f == nil { panic("no future") } - // Capture the evals - all := make([]*structs.Evaluation, 0, len(evals)) + // Create the request + req := &structs.AllocUpdateDesiredTransitionRequest{ + Allocs: allocs, + Evals: make([]*structs.Evaluation, 0, len(evals)), + } + for _, e := range evals { - all = append(all, e) + req.Evals = append(req.Evals, e) } // Upsert the evals in a go routine - go f.Set(b.raft.UpsertEvals(all)) + go f.Set(b.raft.UpdateAllocDesiredTransition(req)) // Reset the evals list and timer evals = make(map[string]*structs.Evaluation) + allocs = make(map[string]*structs.DesiredTransition) timerCh = nil } } } -// EvalFuture is a future that can be used to retrieve the index the eval was +// BatchFuture is a future that can be used to retrieve the index the eval was // created at or any error in the creation process -type EvalFuture struct { +type BatchFuture struct { index uint64 err error waitCh chan struct{} } -// NewEvalFuture returns a new EvalFuture -func NewEvalFuture() *EvalFuture { - return &EvalFuture{ +// NewBatchFuture returns a new BatchFuture +func NewBatchFuture() *BatchFuture { + return &BatchFuture{ waitCh: make(chan struct{}), } } // Set sets the results of the future, unblocking any client. -func (f *EvalFuture) Set(index uint64, err error) { +func (f *BatchFuture) Set(index uint64, err error) { f.index = index f.err = err close(f.waitCh) } // Results returns the creation index and any error. -func (f *EvalFuture) Results() (uint64, error) { +func (f *BatchFuture) Results() (uint64, error) { <-f.waitCh return f.index, f.err } diff --git a/nomad/deploymentwatcher/deployment_watcher.go b/nomad/deploymentwatcher/deployment_watcher.go index 80cf9737d82c..f8c9e32471a2 100644 --- a/nomad/deploymentwatcher/deployment_watcher.go +++ b/nomad/deploymentwatcher/deployment_watcher.go @@ -2,6 +2,7 @@ package deploymentwatcher import ( "context" + "fmt" "log" "sync" "time" @@ -21,11 +22,21 @@ const ( perJobEvalBatchPeriod = 1 * time.Second ) +var ( + // allowRescheduleTransition is the transition that allows failed + // allocations part of a deployment to be rescheduled. We create a one off + // variable to avoid creating a new object for every request. + allowRescheduleTransition = &structs.DesiredTransition{ + Reschedule: helper.BoolToPtr(true), + } +) + // deploymentTriggers are the set of functions required to trigger changes on // behalf of a deployment type deploymentTriggers interface { - // createEvaluation is used to create an evaluation. - createEvaluation(eval *structs.Evaluation) (uint64, error) + // createUpdate is used to create allocation desired transition updates and + // an evaluation. + createUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) (uint64, error) // upsertJob is used to roll back a job when autoreverting for a deployment upsertJob(job *structs.Job) (uint64, error) @@ -55,6 +66,12 @@ type deploymentWatcher struct { // state is the state that is watched for state changes. state *state.StateStore + // deploymentID is the deployment's ID being watched + deploymentID string + + // deploymentUpdateCh is triggered when there is an updated deployment + deploymentUpdateCh chan struct{} + // d is the deployment being watched d *structs.Deployment @@ -62,9 +79,13 @@ type deploymentWatcher struct { j *structs.Job // outstandingBatch marks whether an outstanding function exists to create - // the evaluation. Access should be done through the lock + // the evaluation. Access should be done through the lock. outstandingBatch bool + // outstandingAllowReplacements is the map of allocations that will be + // marked as allowing a replacement. Access should be done through the lock. + outstandingAllowReplacements map[string]*structs.DesiredTransition + // latestEval is the latest eval for the job. It is updated by the watch // loop and any time an evaluation is created. The field should be accessed // by holding the lock or using the setter and getter methods. @@ -85,6 +106,8 @@ func newDeploymentWatcher(parent context.Context, queryLimiter *rate.Limiter, ctx, exitFn := context.WithCancel(parent) w := &deploymentWatcher{ queryLimiter: queryLimiter, + deploymentID: d.ID, + deploymentUpdateCh: make(chan struct{}, 1), d: d, j: j, state: state, @@ -100,6 +123,26 @@ func newDeploymentWatcher(parent context.Context, queryLimiter *rate.Limiter, return w } +// updateDeployment is used to update the tracked deployment. +func (w *deploymentWatcher) updateDeployment(d *structs.Deployment) { + w.l.Lock() + defer w.l.Unlock() + + // Update and trigger + w.d = d + select { + case w.deploymentUpdateCh <- struct{}{}: + default: + } +} + +// getDeployment returns the tracked deployment. +func (w *deploymentWatcher) getDeployment() *structs.Deployment { + w.l.RLock() + defer w.l.RUnlock() + return w.d +} + func (w *deploymentWatcher) SetAllocHealth( req *structs.DeploymentAllocHealthRequest, resp *structs.DeploymentUpdateResponse) error { @@ -137,7 +180,7 @@ func (w *deploymentWatcher) SetAllocHealth( } // Check if the group has autorevert set - group, ok := w.d.TaskGroups[alloc.TaskGroup] + group, ok := w.getDeployment().TaskGroups[alloc.TaskGroup] if !ok || !group.AutoRevert { continue } @@ -163,9 +206,10 @@ func (w *deploymentWatcher) SetAllocHealth( // Create the request areq := &structs.ApplyDeploymentAllocHealthRequest{ DeploymentAllocHealthRequest: *req, - Eval: w.getEval(), - DeploymentUpdate: u, - Job: j, + Timestamp: time.Now(), + Eval: w.getEval(), + DeploymentUpdate: u, + Job: j, } index, err := w.upsertDeploymentAllocHealth(areq) @@ -264,7 +308,7 @@ func (w *deploymentWatcher) FailDeployment( // Determine if we should rollback rollback := false - for _, state := range w.d.TaskGroups { + for _, state := range w.getDeployment().TaskGroups { if state.AutoRevert { rollback = true break @@ -312,100 +356,273 @@ func (w *deploymentWatcher) StopWatch() { w.exitFn() } -// watch is the long running watcher that takes actions upon allocation changes +// watch is the long running watcher that watches for both allocation and +// deployment changes. Its function is to create evaluations to trigger the +// scheduler when more progress can be made, to fail the deployment if it has +// failed and potentially rolling back the job. Progress can be made when an +// allocation transitions to healthy, so we create an eval. func (w *deploymentWatcher) watch() { + // Get the deadline. This is likely a zero time to begin with but we need to + // handle the case that the deployment has already progressed and we are now + // just starting to watch it. This must likely would occur if there was a + // leader transition and we are now starting our watcher. + currentDeadline := getDeploymentProgressCutoff(w.getDeployment()) + var deadlineTimer *time.Timer + if currentDeadline.IsZero() { + deadlineTimer = time.NewTimer(0) + if !deadlineTimer.Stop() { + <-deadlineTimer.C + } + } else { + deadlineTimer = time.NewTimer(currentDeadline.Sub(time.Now())) + } + allocIndex := uint64(1) + var updates *allocUpdates + + rollback, deadlineHit := false, false + +FAIL: for { - // Block getting all allocations that are part of the deployment using - // the last evaluation index. This will have us block waiting for - // something to change past what the scheduler has evaluated. - allocs, index, err := w.getAllocs(allocIndex) - if err != nil { - if err == context.Canceled || w.ctx.Err() == context.Canceled { + select { + case <-w.ctx.Done(): + return + case <-deadlineTimer.C: + // We have hit the progress deadline so fail the deployment. We need + // to determine whether we should roll back the job by inspecting + // which allocs as part of the deployment are healthy and which + // aren't. + deadlineHit = true + fail, rback, err := w.shouldFail() + if err != nil { + w.logger.Printf("[ERR] nomad.deployment_watcher: failed to determine whether to rollback job for deployment %q: %v", w.deploymentID, err) + } + if !fail { + w.logger.Printf("[DEBUG] nomad.deployment_watcher: skipping deadline for deployment %q", w.deploymentID) + continue + } + + w.logger.Printf("[DEBUG] nomad.deployment_watcher: deadline for deployment %q hit and rollback is %v", w.deploymentID, rback) + rollback = rback + break FAIL + case <-w.deploymentUpdateCh: + // Get the updated deployment and check if we should change the + // deadline timer + next := getDeploymentProgressCutoff(w.getDeployment()) + if !next.Equal(currentDeadline) { + prevDeadlineZero := currentDeadline.IsZero() + currentDeadline = next + // The most recent deadline can be zero if no allocs were created for this deployment. + // The deadline timer would have already been stopped once in that case. To prevent + // deadlocking on the already stopped deadline timer, we only drain the channel if + // the previous deadline was not zero. + if !prevDeadlineZero && !deadlineTimer.Stop() { + select { + case <-deadlineTimer.C: + default: + } + } + deadlineTimer.Reset(next.Sub(time.Now())) + } + + case updates = <-w.getAllocsCh(allocIndex): + if err := updates.err; err != nil { + if err == context.Canceled || w.ctx.Err() == context.Canceled { + return + } + + w.logger.Printf("[ERR] nomad.deployment_watcher: failed to retrieve allocations for deployment %q: %v", w.deploymentID, err) return } + allocIndex = updates.index - w.logger.Printf("[ERR] nomad.deployment_watcher: failed to retrieve allocations for deployment %q: %v", w.d.ID, err) - return - } - allocIndex = index + // We have allocation changes for this deployment so determine the + // steps to take. + res, err := w.handleAllocUpdate(updates.allocs) + if err != nil { + if err == context.Canceled || w.ctx.Err() == context.Canceled { + return + } - // Get the latest evaluation index - latestEval, err := w.latestEvalIndex() - if err != nil { - if err == context.Canceled || w.ctx.Err() == context.Canceled { + w.logger.Printf("[ERR] nomad.deployment_watcher: failed handling allocation updates: %v", err) return } - w.logger.Printf("[ERR] nomad.deployment_watcher: failed to determine last evaluation index for job %q: %v", w.d.JobID, err) - return - } + // The deployment has failed, so break out of the watch loop and + // handle the failure + if res.failDeployment { + rollback = res.rollback + break FAIL + } - // Create an evaluation trigger if there is any allocation whose - // deployment status has been updated past the latest eval index. - createEval, failDeployment, rollback := false, false, false - for _, alloc := range allocs { - if alloc.DeploymentStatus == nil || alloc.DeploymentStatus.ModifyIndex <= latestEval { - continue + // Create an eval to push the deployment along + if res.createEval || len(res.allowReplacements) != 0 { + w.createBatchedUpdate(res.allowReplacements, allocIndex) } + } + } + + // Change the deployments status to failed + desc := structs.DeploymentStatusDescriptionFailedAllocations + if deadlineHit { + desc = structs.DeploymentStatusDescriptionProgressDeadline + } - // We need to create an eval - createEval = true + // Rollback to the old job if necessary + var j *structs.Job + if rollback { + var err error + j, err = w.latestStableJob() + if err != nil { + w.logger.Printf("[ERR] nomad.deployment_watcher: failed to lookup latest stable job for %q: %v", w.j.ID, err) + } - if alloc.DeploymentStatus.IsUnhealthy() { - // Check if the group has autorevert set - group, ok := w.d.TaskGroups[alloc.TaskGroup] - if ok && group.AutoRevert { - rollback = true - } + // Description should include that the job is being rolled back to + // version N + if j != nil { + j, desc = w.handleRollbackValidity(j, desc) + } else { + desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc) + } + } - // Since we have an unhealthy allocation, fail the deployment - failDeployment = true - } + // Update the status of the deployment to failed and create an evaluation. + e := w.getEval() + u := w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc) + if index, err := w.upsertDeploymentStatusUpdate(u, e, j); err != nil { + w.logger.Printf("[ERR] nomad.deployment_watcher: failed to update deployment %q status: %v", w.deploymentID, err) + } else { + w.setLatestEval(index) + } +} - // All conditions have been hit so we can break - if createEval && failDeployment && rollback { - break - } +// allocUpdateResult is used to return the desired actions given the newest set +// of allocations for the deployment. +type allocUpdateResult struct { + createEval bool + failDeployment bool + rollback bool + allowReplacements []string +} + +// handleAllocUpdate is used to compute the set of actions to take based on the +// updated allocations for the deployment. +func (w *deploymentWatcher) handleAllocUpdate(allocs []*structs.AllocListStub) (allocUpdateResult, error) { + var res allocUpdateResult + + // Get the latest evaluation index + latestEval, err := w.latestEvalIndex() + if err != nil { + if err == context.Canceled || w.ctx.Err() == context.Canceled { + return res, err } - // Change the deployments status to failed - if failDeployment { - // Default description - desc := structs.DeploymentStatusDescriptionFailedAllocations - - // Rollback to the old job if necessary - var j *structs.Job - if rollback { - var err error - j, err = w.latestStableJob() - if err != nil { - w.logger.Printf("[ERR] nomad.deployment_watcher: failed to lookup latest stable job for %q: %v", w.d.JobID, err) - } + return res, fmt.Errorf("failed to determine last evaluation index for job %q: %v", w.j.ID, err) + } - // Description should include that the job is being rolled back to - // version N - if j != nil { - j, desc = w.handleRollbackValidity(j, desc) - } else { - desc = structs.DeploymentStatusDescriptionNoRollbackTarget(desc) - } + deployment := w.getDeployment() + for _, alloc := range allocs { + dstate, ok := deployment.TaskGroups[alloc.TaskGroup] + if !ok { + continue + } + + // Nothing to do for this allocation + if alloc.DeploymentStatus == nil || alloc.DeploymentStatus.ModifyIndex <= latestEval { + continue + } + + // Determine if the update stanza for this group is progress based + progressBased := dstate.ProgressDeadline != 0 + + // We need to create an eval so the job can progress. + if alloc.DeploymentStatus.IsHealthy() { + res.createEval = true + } else if progressBased && alloc.DeploymentStatus.IsUnhealthy() && deployment.Active() && !alloc.DesiredTransition.ShouldReschedule() { + res.allowReplacements = append(res.allowReplacements, alloc.ID) + } + + // If the group is using a progress deadline, we don't have to do anything. + if progressBased { + continue + } + + // Fail on the first bad allocation + if alloc.DeploymentStatus.IsUnhealthy() { + // Check if the group has autorevert set + if dstate.AutoRevert { + res.rollback = true } - // Update the status of the deployment to failed and create an - // evaluation. - e := w.getEval() - u := w.getDeploymentStatusUpdate(structs.DeploymentStatusFailed, desc) - if index, err := w.upsertDeploymentStatusUpdate(u, e, j); err != nil { - w.logger.Printf("[ERR] nomad.deployment_watcher: failed to update deployment %q status: %v", w.d.ID, err) - } else { - w.setLatestEval(index) + // Since we have an unhealthy allocation, fail the deployment + res.failDeployment = true + } + + // All conditions have been hit so we can break + if res.createEval && res.failDeployment && res.rollback { + break + } + } + + return res, nil +} + +// shouldFail returns whether the job should be failed and whether it should +// rolled back to an earlier stable version by examining the allocations in the +// deployment. +func (w *deploymentWatcher) shouldFail() (fail, rollback bool, err error) { + snap, err := w.state.Snapshot() + if err != nil { + return false, false, err + } + + d, err := snap.DeploymentByID(nil, w.deploymentID) + if err != nil { + return false, false, err + } + if d == nil { + // The deployment wasn't in the state store, possibly due to a system gc + return false, false, fmt.Errorf("deployment id not found: %q", w.deploymentID) + } + + fail = false + for tg, state := range d.TaskGroups { + // If we are in a canary state we fail if there aren't enough healthy + // allocs to satisfy DesiredCanaries + if state.DesiredCanaries > 0 && !state.Promoted { + if state.HealthyAllocs >= state.DesiredCanaries { + continue } - } else if createEval { - // Create an eval to push the deployment along - w.createEvalBatched(index) + } else if state.HealthyAllocs >= state.DesiredTotal { + continue + } + + // We have failed this TG + fail = true + + // We don't need to autorevert this group + upd := w.j.LookupTaskGroup(tg).Update + if upd == nil || !upd.AutoRevert { + continue + } + + // Unhealthy allocs and we need to autorevert + return true, true, nil + } + + return fail, false, nil +} + +// getDeploymentProgressCutoff returns the progress cutoff for the given +// deployment +func getDeploymentProgressCutoff(d *structs.Deployment) time.Time { + var next time.Time + for _, state := range d.TaskGroups { + if next.IsZero() || state.RequireProgressBy.Before(next) { + next = state.RequireProgressBy } } + return next } // latestStableJob returns the latest stable job. It may be nil if none exist @@ -415,7 +632,7 @@ func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) { return nil, err } - versions, err := snap.JobVersionsByID(nil, w.d.Namespace, w.d.JobID) + versions, err := snap.JobVersionsByID(nil, w.j.Namespace, w.j.ID) if err != nil { return nil, err } @@ -431,12 +648,21 @@ func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) { return stable, nil } -// createEvalBatched creates an eval but batches calls together -func (w *deploymentWatcher) createEvalBatched(forIndex uint64) { +// createBatchedUpdate creates an eval for the given index as well as updating +// the given allocations to allow them to reschedule. +func (w *deploymentWatcher) createBatchedUpdate(allowReplacements []string, forIndex uint64) { w.l.Lock() defer w.l.Unlock() - if w.outstandingBatch || forIndex < w.latestEval { + // Store the allocations that can be replaced + for _, allocID := range allowReplacements { + if w.outstandingAllowReplacements == nil { + w.outstandingAllowReplacements = make(map[string]*structs.DesiredTransition, len(allowReplacements)) + } + w.outstandingAllowReplacements[allocID] = allowRescheduleTransition + } + + if w.outstandingBatch || (forIndex < w.latestEval && len(allowReplacements) == 0) { return } @@ -451,18 +677,18 @@ func (w *deploymentWatcher) createEvalBatched(forIndex uint64) { default: } - // Create the eval - evalCreateIndex, err := w.createEvaluation(w.getEval()) - if err != nil { - w.logger.Printf("[ERR] nomad.deployment_watcher: failed to create evaluation for deployment %q: %v", w.d.ID, err) - } else { - w.setLatestEval(evalCreateIndex) - } - w.l.Lock() + replacements := w.outstandingAllowReplacements + w.outstandingAllowReplacements = nil w.outstandingBatch = false w.l.Unlock() + // Create the eval + if index, err := w.createUpdate(replacements, w.getEval()); err != nil { + w.logger.Printf("[ERR] nomad.deployment_watcher: failed to create evaluation for deployment %q: %v", w.deploymentID, err) + } else { + w.setLatestEval(index) + } }) } @@ -475,7 +701,7 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation { Type: w.j.Type, TriggeredBy: structs.EvalTriggerDeploymentWatcher, JobID: w.j.ID, - DeploymentID: w.d.ID, + DeploymentID: w.deploymentID, Status: structs.EvalStatusPending, } } @@ -483,12 +709,34 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation { // getDeploymentStatusUpdate returns a deployment status update func (w *deploymentWatcher) getDeploymentStatusUpdate(status, desc string) *structs.DeploymentStatusUpdate { return &structs.DeploymentStatusUpdate{ - DeploymentID: w.d.ID, + DeploymentID: w.deploymentID, Status: status, StatusDescription: desc, } } +type allocUpdates struct { + allocs []*structs.AllocListStub + index uint64 + err error +} + +// getAllocsCh retrieves the allocations that are part of the deployment blocking +// at the given index. +func (w *deploymentWatcher) getAllocsCh(index uint64) <-chan *allocUpdates { + out := make(chan *allocUpdates, 1) + go func() { + allocs, index, err := w.getAllocs(index) + out <- &allocUpdates{ + allocs: allocs, + index: index, + err: err, + } + }() + + return out +} + // getAllocs retrieves the allocations that are part of the deployment blocking // at the given index. func (w *deploymentWatcher) getAllocs(index uint64) ([]*structs.AllocListStub, uint64, error) { @@ -510,7 +758,7 @@ func (w *deploymentWatcher) getAllocsImpl(ws memdb.WatchSet, state *state.StateS } // Capture all the allocations - allocs, err := state.AllocsByDeployment(ws, w.d.ID) + allocs, err := state.AllocsByDeployment(ws, w.deploymentID) if err != nil { return nil, 0, err } @@ -542,7 +790,7 @@ func (w *deploymentWatcher) latestEvalIndex() (uint64, error) { return 0, err } - evals, err := snap.EvalsByJob(nil, w.d.Namespace, w.d.JobID) + evals, err := snap.EvalsByJob(nil, w.j.Namespace, w.j.ID) if err != nil { return 0, err } @@ -552,6 +800,7 @@ func (w *deploymentWatcher) latestEvalIndex() (uint64, error) { if err != nil { w.setLatestEval(idx) } + return idx, err } diff --git a/nomad/deploymentwatcher/deployments_watcher.go b/nomad/deploymentwatcher/deployments_watcher.go index a88a1de67f93..d4a1eb7bbf1a 100644 --- a/nomad/deploymentwatcher/deployments_watcher.go +++ b/nomad/deploymentwatcher/deployments_watcher.go @@ -19,9 +19,10 @@ const ( // second LimitStateQueriesPerSecond = 100.0 - // CrossDeploymentEvalBatchDuration is the duration in which evaluations are - // batched across all deployment watchers before committing to Raft. - CrossDeploymentEvalBatchDuration = 250 * time.Millisecond + // CrossDeploymentUpdateBatchDuration is the duration in which allocation + // desired transition and evaluation creation updates are batched across + // all deployment watchers before committing to Raft. + CrossDeploymentUpdateBatchDuration = 250 * time.Millisecond ) var ( @@ -33,9 +34,6 @@ var ( // DeploymentRaftEndpoints exposes the deployment watcher to a set of functions // to apply data transforms via Raft. type DeploymentRaftEndpoints interface { - // UpsertEvals is used to upsert a set of evaluations - UpsertEvals([]*structs.Evaluation) (uint64, error) - // UpsertJob is used to upsert a job UpsertJob(job *structs.Job) (uint64, error) @@ -49,6 +47,10 @@ type DeploymentRaftEndpoints interface { // UpdateDeploymentAllocHealth is used to set the health of allocations in a // deployment UpdateDeploymentAllocHealth(req *structs.ApplyDeploymentAllocHealthRequest) (uint64, error) + + // UpdateAllocDesiredTransition is used to update the desired transition + // for allocations. + UpdateAllocDesiredTransition(req *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) } // Watcher is used to watch deployments and their allocations created @@ -61,9 +63,9 @@ type Watcher struct { // queryLimiter is used to limit the rate of blocking queries queryLimiter *rate.Limiter - // evalBatchDuration is the duration to batch eval creation across all - // deployment watchers - evalBatchDuration time.Duration + // updateBatchDuration is the duration to batch allocation desired + // transition and eval creation across all deployment watchers + updateBatchDuration time.Duration // raft contains the set of Raft endpoints that can be used by the // deployments watcher @@ -75,8 +77,9 @@ type Watcher struct { // watchers is the set of active watchers, one per deployment watchers map[string]*deploymentWatcher - // evalBatcher is used to batch the creation of evaluations - evalBatcher *EvalBatcher + // allocUpdateBatcher is used to batch the creation of evaluations and + // allocation desired transition updates + allocUpdateBatcher *AllocUpdateBatcher // ctx and exitFn are used to cancel the watcher ctx context.Context @@ -89,13 +92,13 @@ type Watcher struct { // deployments and trigger the scheduler as needed. func NewDeploymentsWatcher(logger *log.Logger, raft DeploymentRaftEndpoints, stateQueriesPerSecond float64, - evalBatchDuration time.Duration) *Watcher { + updateBatchDuration time.Duration) *Watcher { return &Watcher{ - raft: raft, - queryLimiter: rate.NewLimiter(rate.Limit(stateQueriesPerSecond), 100), - evalBatchDuration: evalBatchDuration, - logger: logger, + raft: raft, + queryLimiter: rate.NewLimiter(rate.Limit(stateQueriesPerSecond), 100), + updateBatchDuration: updateBatchDuration, + logger: logger, } } @@ -136,7 +139,7 @@ func (w *Watcher) flush() { w.watchers = make(map[string]*deploymentWatcher, 32) w.ctx, w.exitFn = context.WithCancel(context.Background()) - w.evalBatcher = NewEvalBatcher(w.evalBatchDuration, w.raft, w.ctx) + w.allocUpdateBatcher = NewAllocUpdateBatcher(w.updateBatchDuration, w.raft, w.ctx) } // watchDeployments is the long lived go-routine that watches for deployments to @@ -228,8 +231,9 @@ func (w *Watcher) addLocked(d *structs.Deployment) (*deploymentWatcher, error) { return nil, fmt.Errorf("deployment %q is terminal", d.ID) } - // Already watched so no-op - if _, ok := w.watchers[d.ID]; ok { + // Already watched so just update the deployment + if w, ok := w.watchers[d.ID]; ok { + w.updateDeployment(d) return nil, nil } @@ -353,10 +357,10 @@ func (w *Watcher) FailDeployment(req *structs.DeploymentFailRequest, resp *struc return watcher.FailDeployment(req, resp) } -// createEvaluation commits the given evaluation to Raft but batches the commit -// with other calls. -func (w *Watcher) createEvaluation(eval *structs.Evaluation) (uint64, error) { - return w.evalBatcher.CreateEval(eval).Results() +// createUpdate commits the given allocation desired transition and evaluation +// to Raft but batches the commit with other calls. +func (w *Watcher) createUpdate(allocs map[string]*structs.DesiredTransition, eval *structs.Evaluation) (uint64, error) { + return w.allocUpdateBatcher.CreateUpdate(allocs, eval).Results() } // upsertJob commits the given job to Raft diff --git a/nomad/deploymentwatcher/deployments_watcher_test.go b/nomad/deploymentwatcher/deployments_watcher_test.go index 7ff516257140..1221a7a04396 100644 --- a/nomad/deploymentwatcher/deployments_watcher_test.go +++ b/nomad/deploymentwatcher/deployments_watcher_test.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" mocker "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (*Watcher, *mockBackend) { @@ -22,7 +23,7 @@ func testDeploymentWatcher(t *testing.T, qps float64, batchDur time.Duration) (* } func defaultTestDeploymentWatcher(t *testing.T) (*Watcher, *mockBackend) { - return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentEvalBatchDuration) + return testDeploymentWatcher(t, LimitStateQueriesPerSecond, CrossDeploymentUpdateBatchDuration) } // Tests that the watcher properly watches for deployments and reconciles them @@ -141,10 +142,6 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentAllocHealth a := mock.Alloc() matchConfig := &matchDeploymentAllocHealthRequestConfig{ @@ -155,6 +152,10 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) { matcher := matchDeploymentAllocHealthRequest(matchConfig) m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentAllocHealthRequest{ DeploymentID: d.ID, @@ -184,10 +185,6 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) { assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentAllocHealth matchConfig := &matchDeploymentAllocHealthRequestConfig{ DeploymentID: d.ID, @@ -197,6 +194,10 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) { matcher := matchDeploymentAllocHealthRequest(matchConfig) m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentAllocHealthRequest{ DeploymentID: d.ID, @@ -225,10 +226,6 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) { assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentAllocHealth matchConfig := &matchDeploymentAllocHealthRequestConfig{ DeploymentID: d.ID, @@ -243,6 +240,10 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) { matcher := matchDeploymentAllocHealthRequest(matchConfig) m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentAllocHealthRequest{ DeploymentID: d.ID, @@ -268,6 +269,7 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 j.TaskGroups[0].Update.AutoRevert = true + j.TaskGroups[0].Update.ProgressDeadline = 0 j.Stable = true d := mock.Deployment() d.JobID = j.ID @@ -286,10 +288,6 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentAllocHealth matchConfig := &matchDeploymentAllocHealthRequestConfig{ DeploymentID: d.ID, @@ -305,6 +303,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { matcher := matchDeploymentAllocHealthRequest(matchConfig) m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentAllocHealthRequest{ DeploymentID: d.ID, @@ -330,6 +332,7 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) { j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 j.TaskGroups[0].Update.AutoRevert = true + j.TaskGroups[0].Update.ProgressDeadline = 0 j.Stable = true d := mock.Deployment() d.JobID = j.ID @@ -346,10 +349,6 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentAllocHealth matchConfig := &matchDeploymentAllocHealthRequestConfig{ DeploymentID: d.ID, @@ -365,6 +364,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) { matcher := matchDeploymentAllocHealthRequest(matchConfig) m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentAllocHealthRequest{ DeploymentID: d.ID, @@ -389,10 +392,12 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 - j.TaskGroups[0].Update.Canary = 2 + j.TaskGroups[0].Update.Canary = 1 + j.TaskGroups[0].Update.ProgressDeadline = 0 d := mock.Deployment() d.JobID = j.ID a := mock.Alloc() + d.TaskGroups[a.TaskGroup].DesiredCanaries = 1 d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} a.DeploymentStatus = &structs.AllocDeploymentStatus{ Healthy: helper.BoolToPtr(true), @@ -402,10 +407,6 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentPromotion matchConfig := &matchDeploymentPromoteRequestConfig{ Promotion: &structs.DeploymentPromoteRequest{ @@ -417,6 +418,14 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { matcher := matchDeploymentPromoteRequest(matchConfig) m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil) + // We may get an update for the desired transition. + m1 := matchUpdateAllocDesiredTransitions([]string{d.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PromoteDeployment req := &structs.DeploymentPromoteRequest{ DeploymentID: d.ID, @@ -440,19 +449,17 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) { j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 j.TaskGroups[0].Update.Canary = 2 + j.TaskGroups[0].Update.ProgressDeadline = 0 d := mock.Deployment() d.JobID = j.ID a := mock.Alloc() d.TaskGroups[a.TaskGroup].PlacedCanaries = []string{a.ID} + d.TaskGroups[a.TaskGroup].DesiredCanaries = 2 a.DeploymentID = d.ID assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentPromotion matchConfig := &matchDeploymentPromoteRequestConfig{ Promotion: &structs.DeploymentPromoteRequest{ @@ -464,6 +471,10 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) { matcher := matchDeploymentPromoteRequest(matchConfig) m.On("UpdateDeploymentPromotion", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call SetAllocHealth req := &structs.DeploymentPromoteRequest{ DeploymentID: d.ID, @@ -472,7 +483,7 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) { var resp structs.DeploymentUpdateResponse err := w.PromoteDeployment(req, &resp) if assert.NotNil(err, "PromoteDeployment") { - assert.Contains(err.Error(), "is not healthy", "Should error because canary isn't marked healthy") + assert.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`, "Should error because canary isn't marked healthy") } assert.Equal(1, len(w.watchers), "Deployment should still be active") @@ -492,10 +503,6 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentStatusUpdate matchConfig := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, @@ -505,6 +512,10 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) { matcher := matchDeploymentStatusUpdateRequest(matchConfig) m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PauseDeployment req := &structs.DeploymentPauseRequest{ DeploymentID: d.ID, @@ -532,10 +543,6 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentStatusUpdate matchConfig := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, @@ -545,6 +552,10 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) { matcher := matchDeploymentStatusUpdateRequest(matchConfig) m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PauseDeployment req := &structs.DeploymentPauseRequest{ DeploymentID: d.ID, @@ -572,10 +583,6 @@ func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentStatusUpdate matchConfig := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, @@ -586,6 +593,10 @@ func TestWatcher_PauseDeployment_Unpause_Paused(t *testing.T) { matcher := matchDeploymentStatusUpdateRequest(matchConfig) m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PauseDeployment req := &structs.DeploymentPauseRequest{ DeploymentID: d.ID, @@ -612,10 +623,6 @@ func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentStatusUpdate matchConfig := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, @@ -626,6 +633,10 @@ func TestWatcher_PauseDeployment_Unpause_Running(t *testing.T) { matcher := matchDeploymentStatusUpdateRequest(matchConfig) m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PauseDeployment req := &structs.DeploymentPauseRequest{ DeploymentID: d.ID, @@ -652,10 +663,6 @@ func TestWatcher_FailDeployment_Running(t *testing.T) { assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, - func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we get a call to UpsertDeploymentStatusUpdate matchConfig := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, @@ -666,6 +673,10 @@ func TestWatcher_FailDeployment_Running(t *testing.T) { matcher := matchDeploymentStatusUpdateRequest(matchConfig) m.On("UpdateDeploymentStatus", mocker.MatchedBy(matcher)).Return(nil) + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + // Call PauseDeployment req := &structs.DeploymentFailRequest{ DeploymentID: d.ID, @@ -680,7 +691,7 @@ func TestWatcher_FailDeployment_Running(t *testing.T) { // Tests that the watcher properly watches for allocation changes and takes the // proper actions -func TestDeploymentWatcher_Watch(t *testing.T) { +func TestDeploymentWatcher_Watch_NoProgressDeadline(t *testing.T) { t.Parallel() assert := assert.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) @@ -690,6 +701,7 @@ func TestDeploymentWatcher_Watch(t *testing.T) { j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 j.TaskGroups[0].Update.AutoRevert = true + j.TaskGroups[0].Update.ProgressDeadline = 0 j.Stable = true d := mock.Deployment() d.JobID = j.ID @@ -707,15 +719,26 @@ func TestDeploymentWatcher_Watch(t *testing.T) { j2.Stable = false assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") + // Assert that we will get a update allocation call only once. This will + // verify that the watcher is batching allocation changes + m1 := matchUpdateAllocDesiredTransitions([]string{d.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + // Assert that we get a call to UpsertDeploymentStatusUpdate + c := &matchDeploymentStatusUpdateConfig{ + DeploymentID: d.ID, + Status: structs.DeploymentStatusFailed, + StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0), + JobVersion: helper.Uint64ToPtr(0), + Eval: true, + } + m2 := matchDeploymentStatusUpdateRequest(c) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) + w.SetEnabled(true, m.state) testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we will get a createEvaluation call only once. This will - // verify that the watcher is batching allocation changes - m1 := matchUpsertEvals([]string{d.ID}) - m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() - // Update the allocs health to healthy which should create an evaluation for i := 0; i < 5; i++ { req := &structs.ApplyDeploymentAllocHealthRequest{ @@ -744,17 +767,6 @@ func TestDeploymentWatcher_Watch(t *testing.T) { t.Fatal(err) }) - // Assert that we get a call to UpsertDeploymentStatusUpdate - c := &matchDeploymentStatusUpdateConfig{ - DeploymentID: d.ID, - Status: structs.DeploymentStatusFailed, - StatusDescription: structs.DeploymentStatusDescriptionRollback(structs.DeploymentStatusDescriptionFailedAllocations, 0), - JobVersion: helper.Uint64ToPtr(0), - Eval: true, - } - m2 := matchDeploymentStatusUpdateRequest(c) - m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) - // Update the allocs health to unhealthy which should create a job rollback, // status update and eval req2 := &structs.ApplyDeploymentAllocHealthRequest{ @@ -782,7 +794,7 @@ func TestDeploymentWatcher_Watch(t *testing.T) { t.Fatal(err) }) - m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) + m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1)) // After we upsert the job version will go to 2. So use this to assert the // original call happened. @@ -799,6 +811,305 @@ func TestDeploymentWatcher_Watch(t *testing.T) { func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) } +func TestDeploymentWatcher_Watch_ProgressDeadline(t *testing.T) { + t.Parallel() + assert := assert.New(t) + w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + + // Create a job, alloc, and a deployment + j := mock.Job() + j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j.TaskGroups[0].Update.MaxParallel = 2 + j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond + j.Stable = true + d := mock.Deployment() + d.JobID = j.ID + d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond + a := mock.Alloc() + now := time.Now() + a.CreateTime = now.UnixNano() + a.ModifyTime = now.UnixNano() + a.DeploymentID = d.ID + assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") + assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") + + // Assert that we get a call to UpsertDeploymentStatusUpdate + c := &matchDeploymentStatusUpdateConfig{ + DeploymentID: d.ID, + Status: structs.DeploymentStatusFailed, + StatusDescription: structs.DeploymentStatusDescriptionProgressDeadline, + Eval: true, + } + m2 := matchDeploymentStatusUpdateRequest(c) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + + // Update the alloc to be unhealthy and assert that nothing happens. + a2 := a.Copy() + a2.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(false), + Timestamp: now, + } + assert.Nil(m.state.UpdateAllocsFromClient(100, []*structs.Allocation{a2})) + + // Wait for the deployment to be failed + testutil.WaitForResult(func() (bool, error) { + d, err := m.state.DeploymentByID(nil, d.ID) + if err != nil { + return false, err + } + + return d.Status == structs.DeploymentStatusFailed, fmt.Errorf("bad status %q", d.Status) + }, func(err error) { + t.Fatal(err) + }) + + // Assert there are is only one evaluation + testutil.WaitForResult(func() (bool, error) { + ws := memdb.NewWatchSet() + evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID) + if err != nil { + return false, err + } + + if l := len(evals); l != 1 { + return false, fmt.Errorf("Got %d evals; want 1", l) + } + + return true, nil + }, func(err error) { + t.Fatal(err) + }) +} + +// Test that we will allow the progress deadline to be reached when the canaries +// are healthy but we haven't promoted +func TestDeploymentWatcher_Watch_ProgressDeadline_Canaries(t *testing.T) { + t.Parallel() + require := require.New(t) + w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + + // Create a job, alloc, and a deployment + j := mock.Job() + j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j.TaskGroups[0].Update.Canary = 1 + j.TaskGroups[0].Update.MaxParallel = 1 + j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond + j.Stable = true + d := mock.Deployment() + d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion + d.JobID = j.ID + d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond + d.TaskGroups["web"].DesiredCanaries = 1 + a := mock.Alloc() + now := time.Now() + a.CreateTime = now.UnixNano() + a.ModifyTime = now.UnixNano() + a.DeploymentID = d.ID + require.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") + require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") + + // Assert that we will get a createEvaluation call only once. This will + // verify that the watcher is batching allocation changes + m1 := matchUpdateAllocDesiredTransitions([]string{d.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { require.Equal(1, len(w.watchers), "Should have 1 deployment") }) + + // Update the alloc to be unhealthy and require that nothing happens. + a2 := a.Copy() + a2.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + Timestamp: now, + } + require.Nil(m.state.UpdateAllocsFromClient(m.nextIndex(), []*structs.Allocation{a2})) + + // Wait for the deployment to cross the deadline + dout, err := m.state.DeploymentByID(nil, d.ID) + require.NoError(err) + require.NotNil(dout) + state := dout.TaskGroups["web"] + require.NotNil(state) + time.Sleep(state.RequireProgressBy.Add(time.Second).Sub(now)) + + // Require the deployment is still running + dout, err = m.state.DeploymentByID(nil, d.ID) + require.NoError(err) + require.NotNil(dout) + require.Equal(structs.DeploymentStatusRunning, dout.Status) + require.Equal(structs.DeploymentStatusDescriptionRunningNeedsPromotion, dout.StatusDescription) + + // require there are is only one evaluation + testutil.WaitForResult(func() (bool, error) { + ws := memdb.NewWatchSet() + evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID) + if err != nil { + return false, err + } + + if l := len(evals); l != 1 { + return false, fmt.Errorf("Got %d evals; want 1", l) + } + + return true, nil + }, func(err error) { + t.Fatal(err) + }) +} + +// Test that a promoted deployment with alloc healthy updates create +// evals to move the deployment forward +func TestDeploymentWatcher_PromotedCanary_UpdatedAllocs(t *testing.T) { + t.Parallel() + require := require.New(t) + w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + + // Create a job, alloc, and a deployment + j := mock.Job() + j.TaskGroups[0].Count = 2 + j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j.TaskGroups[0].Update.Canary = 1 + j.TaskGroups[0].Update.MaxParallel = 1 + j.TaskGroups[0].Update.ProgressDeadline = 50 * time.Millisecond + j.Stable = true + + d := mock.Deployment() + d.TaskGroups["web"].DesiredTotal = 2 + d.TaskGroups["web"].DesiredCanaries = 1 + d.TaskGroups["web"].HealthyAllocs = 1 + d.StatusDescription = structs.DeploymentStatusDescriptionRunning + d.JobID = j.ID + d.TaskGroups["web"].ProgressDeadline = 50 * time.Millisecond + d.TaskGroups["web"].RequireProgressBy = time.Now().Add(50 * time.Millisecond) + + a := mock.Alloc() + now := time.Now() + a.CreateTime = now.UnixNano() + a.ModifyTime = now.UnixNano() + a.DeploymentID = d.ID + a.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + Timestamp: now, + } + require.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") + require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { require.Equal(1, len(w.watchers), "Should have 1 deployment") }) + + m1 := matchUpdateAllocDesiredTransitions([]string{d.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Twice() + + // Create another alloc + a2 := a.Copy() + a2.ID = uuid.Generate() + now = time.Now() + a2.CreateTime = now.UnixNano() + a2.ModifyTime = now.UnixNano() + a2.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + Timestamp: now, + } + d.TaskGroups["web"].RequireProgressBy = time.Now().Add(2 * time.Second) + require.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + // Wait until batch eval period passes before updating another alloc + time.Sleep(1 * time.Second) + require.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs") + + // Wait for the deployment to cross the deadline + dout, err := m.state.DeploymentByID(nil, d.ID) + require.NoError(err) + require.NotNil(dout) + state := dout.TaskGroups["web"] + require.NotNil(state) + time.Sleep(state.RequireProgressBy.Add(time.Second).Sub(now)) + + // There should be two evals + testutil.WaitForResult(func() (bool, error) { + ws := memdb.NewWatchSet() + evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID) + if err != nil { + return false, err + } + + if l := len(evals); l != 2 { + return false, fmt.Errorf("Got %d evals; want 2", l) + } + + return true, nil + }, func(err error) { + t.Fatal(err) + }) +} + +// Test scenario where deployment initially has no progress deadline +// After the deployment is updated, a failed alloc's DesiredTransition should be set +func TestDeploymentWatcher_Watch_StartWithoutProgressDeadline(t *testing.T) { + t.Parallel() + assert := assert.New(t) + w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + + // Create a job, and a deployment + j := mock.Job() + j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j.TaskGroups[0].Update.MaxParallel = 2 + j.TaskGroups[0].Update.ProgressDeadline = 500 * time.Millisecond + j.Stable = true + d := mock.Deployment() + d.JobID = j.ID + + assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") + assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + + a := mock.Alloc() + a.CreateTime = time.Now().UnixNano() + a.DeploymentID = d.ID + + assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") + + d.TaskGroups["web"].ProgressDeadline = 500 * time.Millisecond + // Update the deployment with a progress deadline + assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") + + // Match on DesiredTransition set to Reschedule for the failed alloc + m1 := matchUpdateAllocDesiredTransitionReschedule([]string{a.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, + func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) + + // Update the alloc to be unhealthy + a2 := a.Copy() + a2.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(false), + Timestamp: time.Now(), + } + assert.Nil(m.state.UpdateAllocsFromClient(m.nextIndex(), []*structs.Allocation{a2})) + + // Wait for the alloc's DesiredState to set reschedule + testutil.WaitForResult(func() (bool, error) { + a, err := m.state.AllocByID(nil, a.ID) + if err != nil { + return false, err + } + dt := a.DesiredTransition + shouldReschedule := dt.Reschedule != nil && *dt.Reschedule + return shouldReschedule, fmt.Errorf("Desired Transition Reschedule should be set but got %v", shouldReschedule) + }, func(err error) { + t.Fatal(err) + }) +} + // Tests that the watcher fails rollback when the spec hasn't changed func TestDeploymentWatcher_RollbackFailed(t *testing.T) { t.Parallel() @@ -810,6 +1121,7 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) { j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() j.TaskGroups[0].Update.MaxParallel = 2 j.TaskGroups[0].Update.AutoRevert = true + j.TaskGroups[0].Update.ProgressDeadline = 0 j.Stable = true d := mock.Deployment() d.JobID = j.ID @@ -826,15 +1138,26 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) { j2.Stable = false assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") + // Assert that we will get a createEvaluation call only once. This will + // verify that the watcher is batching allocation changes + m1 := matchUpdateAllocDesiredTransitions([]string{d.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + // Assert that we get a call to UpsertDeploymentStatusUpdate with roll back failed as the status + c := &matchDeploymentStatusUpdateConfig{ + DeploymentID: d.ID, + Status: structs.DeploymentStatusFailed, + StatusDescription: structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0), + JobVersion: nil, + Eval: true, + } + m2 := matchDeploymentStatusUpdateRequest(c) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) + w.SetEnabled(true, m.state) testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) - // Assert that we will get a createEvaluation call only once. This will - // verify that the watcher is batching allocation changes - m1 := matchUpsertEvals([]string{d.ID}) - m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() - // Update the allocs health to healthy which should create an evaluation for i := 0; i < 5; i++ { req := &structs.ApplyDeploymentAllocHealthRequest{ @@ -863,17 +1186,6 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) { t.Fatal(err) }) - // Assert that we get a call to UpsertDeploymentStatusUpdate with roll back failed as the status - c := &matchDeploymentStatusUpdateConfig{ - DeploymentID: d.ID, - Status: structs.DeploymentStatusFailed, - StatusDescription: structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0), - JobVersion: nil, - Eval: true, - } - m2 := matchDeploymentStatusUpdateRequest(c) - m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) - // Update the allocs health to unhealthy which will cause attempting a rollback, // fail in that step, do status update and eval req2 := &structs.ApplyDeploymentAllocHealthRequest{ @@ -901,30 +1213,38 @@ func TestDeploymentWatcher_RollbackFailed(t *testing.T) { t.Fatal(err) }) - m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) + m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1)) // verify that the job version hasn't changed after upsert m.state.JobByID(nil, structs.DefaultNamespace, j.ID) assert.Equal(uint64(0), j.Version, "Expected job version 0 but got ", j.Version) } -// Test evaluations are batched between watchers -func TestWatcher_BatchEvals(t *testing.T) { +// Test allocation updates and evaluation creation is batched between watchers +func TestWatcher_BatchAllocUpdates(t *testing.T) { t.Parallel() assert := assert.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second) // Create a job, alloc, for two deployments j1 := mock.Job() + j1.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j1.TaskGroups[0].Update.ProgressDeadline = 0 d1 := mock.Deployment() d1.JobID = j1.ID a1 := mock.Alloc() + a1.Job = j1 + a1.JobID = j1.ID a1.DeploymentID = d1.ID j2 := mock.Job() + j2.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() + j2.TaskGroups[0].Update.ProgressDeadline = 0 d2 := mock.Deployment() d2.JobID = j2.ID a2 := mock.Alloc() + a2.Job = j2 + a2.JobID = j2.ID a2.DeploymentID = d2.ID assert.Nil(m.state.UpsertJob(m.nextIndex(), j1), "UpsertJob") @@ -934,15 +1254,15 @@ func TestWatcher_BatchEvals(t *testing.T) { assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a1}), "UpsertAllocs") assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a2}), "UpsertAllocs") - w.SetEnabled(true, m.state) - testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, - func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") }) - // Assert that we will get a createEvaluation call only once and it contains // both deployments. This will verify that the watcher is batching // allocation changes - m1 := matchUpsertEvals([]string{d1.ID, d2.ID}) - m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() + m1 := matchUpdateAllocDesiredTransitions([]string{d1.ID, d2.ID}) + m.On("UpdateAllocDesiredTransition", mocker.MatchedBy(m1)).Return(nil).Once() + + w.SetEnabled(true, m.state) + testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, + func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") }) // Update the allocs health to healthy which should create an evaluation req := &structs.ApplyDeploymentAllocHealthRequest{ @@ -975,11 +1295,11 @@ func TestWatcher_BatchEvals(t *testing.T) { } if l := len(evals1); l != 1 { - return false, fmt.Errorf("Got %d evals; want 1", l) + return false, fmt.Errorf("Got %d evals for job %v; want 1", l, j1.ID) } if l := len(evals2); l != 1 { - return false, fmt.Errorf("Got %d evals; want 1", l) + return false, fmt.Errorf("Got %d evals for job 2; want 1", l) } return true, nil @@ -987,7 +1307,7 @@ func TestWatcher_BatchEvals(t *testing.T) { t.Fatal(err) }) - m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) + m.AssertCalled(t, "UpdateAllocDesiredTransition", mocker.MatchedBy(m1)) testutil.WaitForResult(func() (bool, error) { return 2 == len(w.watchers), nil }, func(err error) { assert.Equal(2, len(w.watchers), "Should have 2 deployment") }) } diff --git a/nomad/deploymentwatcher/testutil_test.go b/nomad/deploymentwatcher/testutil_test.go index 98facaff364d..96753a7589b1 100644 --- a/nomad/deploymentwatcher/testutil_test.go +++ b/nomad/deploymentwatcher/testutil_test.go @@ -39,16 +39,16 @@ func (m *mockBackend) nextIndex() uint64 { return i } -func (m *mockBackend) UpsertEvals(evals []*structs.Evaluation) (uint64, error) { - m.Called(evals) +func (m *mockBackend) UpdateAllocDesiredTransition(u *structs.AllocUpdateDesiredTransitionRequest) (uint64, error) { + m.Called(u) i := m.nextIndex() - return i, m.state.UpsertEvals(i, evals) + return i, m.state.UpdateAllocsDesiredTransitions(i, u.Allocs, u.Evals) } -// matchUpsertEvals is used to match an upsert request -func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation) bool { - return func(evals []*structs.Evaluation) bool { - if len(evals) != len(deploymentIDs) { +// matchUpdateAllocDesiredTransitions is used to match an upsert request +func matchUpdateAllocDesiredTransitions(deploymentIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool { + return func(update *structs.AllocUpdateDesiredTransitionRequest) bool { + if len(update.Evals) != len(deploymentIDs) { return false } @@ -57,7 +57,7 @@ func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation) dmap[d] = struct{}{} } - for _, e := range evals { + for _, e := range update.Evals { if _, ok := dmap[e.DeploymentID]; !ok { return false } @@ -69,6 +69,27 @@ func matchUpsertEvals(deploymentIDs []string) func(evals []*structs.Evaluation) } } +// matchUpdateAllocDesiredTransitionReschedule is used to match allocs that have their DesiredTransition set to Reschedule +func matchUpdateAllocDesiredTransitionReschedule(allocIDs []string) func(update *structs.AllocUpdateDesiredTransitionRequest) bool { + return func(update *structs.AllocUpdateDesiredTransitionRequest) bool { + amap := make(map[string]struct{}, len(allocIDs)) + for _, d := range allocIDs { + amap[d] = struct{}{} + } + + for allocID, dt := range update.Allocs { + if _, ok := amap[allocID]; !ok { + return false + } + if !*dt.Reschedule { + return false + } + } + + return true + } +} + func (m *mockBackend) UpsertJob(job *structs.Job) (uint64, error) { m.Called(job) i := m.nextIndex() @@ -196,6 +217,11 @@ func matchDeploymentAllocHealthRequest(c *matchDeploymentAllocHealthRequestConfi return false } + // Require a timestamp + if args.Timestamp.IsZero() { + return false + } + if len(c.Healthy) != len(args.HealthyAllocationIDs) { return false } diff --git a/nomad/drainer/drainer.go b/nomad/drainer/drainer.go index e6be05c7d5b2..841445cb3bae 100644 --- a/nomad/drainer/drainer.go +++ b/nomad/drainer/drainer.go @@ -254,7 +254,7 @@ func (n *NodeDrainer) handleDeadlinedNodes(nodes []string) { n.l.RUnlock() n.batchDrainAllocs(forceStop) - // Submit the node transistions in a sharded form to ensure a reasonable + // Submit the node transitions in a sharded form to ensure a reasonable // Raft transaction size. for _, nodes := range partitionIds(defaultMaxIdsPerTxn, nodes) { if _, err := n.raft.NodesDrainComplete(nodes); err != nil { @@ -324,7 +324,7 @@ func (n *NodeDrainer) handleMigratedAllocs(allocs []*structs.Allocation) { } } - // Submit the node transistions in a sharded form to ensure a reasonable + // Submit the node transitions in a sharded form to ensure a reasonable // Raft transaction size. for _, nodes := range partitionIds(defaultMaxIdsPerTxn, done) { if _, err := n.raft.NodesDrainComplete(nodes); err != nil { @@ -374,9 +374,9 @@ func (n *NodeDrainer) batchDrainAllocs(allocs []*structs.Allocation) (uint64, er func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs.Allocation) { // Compute the effected jobs and make the transition map jobs := make(map[string]*structs.Allocation, 4) - transistions := make(map[string]*structs.DesiredTransition, len(allocs)) + transitions := make(map[string]*structs.DesiredTransition, len(allocs)) for _, alloc := range allocs { - transistions[alloc.ID] = &structs.DesiredTransition{ + transitions[alloc.ID] = &structs.DesiredTransition{ Migrate: helper.BoolToPtr(true), } jobs[alloc.JobID] = alloc @@ -397,7 +397,7 @@ func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs // Commit this update via Raft var finalIndex uint64 - for _, u := range partitionAllocDrain(defaultMaxIdsPerTxn, transistions, evals) { + for _, u := range partitionAllocDrain(defaultMaxIdsPerTxn, transitions, evals) { index, err := n.raft.AllocUpdateDesiredTransition(u.Transitions, u.Evals) if err != nil { future.Respond(0, err) diff --git a/nomad/drainer/drainer_util_test.go b/nomad/drainer/drainer_util_test.go index 057c4e24be86..d18a22d1b4b0 100644 --- a/nomad/drainer/drainer_util_test.go +++ b/nomad/drainer/drainer_util_test.go @@ -13,9 +13,9 @@ func TestDrainer_PartitionAllocDrain(t *testing.T) { maxIdsPerTxn := 2 require := require.New(t) - transistions := map[string]*structs.DesiredTransition{"a": nil, "b": nil, "c": nil} + transitions := map[string]*structs.DesiredTransition{"a": nil, "b": nil, "c": nil} evals := []*structs.Evaluation{nil, nil, nil} - requests := partitionAllocDrain(maxIdsPerTxn, transistions, evals) + requests := partitionAllocDrain(maxIdsPerTxn, transitions, evals) require.Len(requests, 3) first := requests[0] diff --git a/nomad/drainer_int_test.go b/nomad/drainer_int_test.go index 61a6c15870b2..81d8625ca846 100644 --- a/nomad/drainer_int_test.go +++ b/nomad/drainer_int_test.go @@ -38,6 +38,7 @@ func allocPromoter(errCh chan<- error, ctx context.Context, // For each alloc that doesn't have its deployment status set, set it var updates []*structs.Allocation + now := time.Now() for _, alloc := range allocs { if alloc.Job.Type != structs.JobTypeService { continue @@ -48,7 +49,8 @@ func allocPromoter(errCh chan<- error, ctx context.Context, } newAlloc := alloc.Copy() newAlloc.DeploymentStatus = &structs.AllocDeploymentStatus{ - Healthy: helper.BoolToPtr(true), + Healthy: helper.BoolToPtr(true), + Timestamp: now, } updates = append(updates, newAlloc) logger.Printf("Marked deployment health for alloc %q", alloc.ID) @@ -824,7 +826,7 @@ func TestDrainer_AllTypes_Deadline_GarbageCollectedNode(t *testing.T) { }) } -// Test that transistions to force drain work. +// Test that transitions to force drain work. func TestDrainer_Batch_TransitionToForce(t *testing.T) { t.Parallel() require := require.New(t) diff --git a/nomad/fsm.go b/nomad/fsm.go index 4ab6dd108e60..cdbe9207c186 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -582,19 +582,34 @@ func (n *nomadFSM) upsertEvals(index uint64, evals []*structs.Evaluation) error return err } + n.handleUpsertedEvals(evals) + return nil +} + +// handleUpsertingEval is a helper for taking action after upserting +// evaluations. +func (n *nomadFSM) handleUpsertedEvals(evals []*structs.Evaluation) { for _, eval := range evals { - if eval.ShouldEnqueue() { - n.evalBroker.Enqueue(eval) - } else if eval.ShouldBlock() { - n.blockedEvals.Block(eval) - } else if eval.Status == structs.EvalStatusComplete && - len(eval.FailedTGAllocs) == 0 { - // If we have a successful evaluation for a node, untrack any - // blocked evaluation - n.blockedEvals.Untrack(eval.JobID) - } + n.handleUpsertedEval(eval) + } +} + +// handleUpsertingEval is a helper for taking action after upserting an eval. +func (n *nomadFSM) handleUpsertedEval(eval *structs.Evaluation) { + if eval == nil { + return + } + + if eval.ShouldEnqueue() { + n.evalBroker.Enqueue(eval) + } else if eval.ShouldBlock() { + n.blockedEvals.Block(eval) + } else if eval.Status == structs.EvalStatusComplete && + len(eval.FailedTGAllocs) == 0 { + // If we have a successful evaluation for a node, untrack any + // blocked evaluation + n.blockedEvals.Untrack(eval.JobID) } - return nil } func (n *nomadFSM) applyDeleteEval(buf []byte, index uint64) interface{} { @@ -731,10 +746,7 @@ func (n *nomadFSM) applyAllocUpdateDesiredTransition(buf []byte, index uint64) i return err } - if err := n.upsertEvals(index, req.Evals); err != nil { - n.logger.Printf("[ERR] nomad.fsm: AllocUpdateDesiredTransition failed to upsert %d eval(s): %v", len(req.Evals), err) - return err - } + n.handleUpsertedEvals(req.Evals) return nil } @@ -826,10 +838,7 @@ func (n *nomadFSM) applyDeploymentStatusUpdate(buf []byte, index uint64) interfa return err } - if req.Eval != nil && req.Eval.ShouldEnqueue() { - n.evalBroker.Enqueue(req.Eval) - } - + n.handleUpsertedEval(req.Eval) return nil } @@ -846,10 +855,7 @@ func (n *nomadFSM) applyDeploymentPromotion(buf []byte, index uint64) interface{ return err } - if req.Eval != nil && req.Eval.ShouldEnqueue() { - n.evalBroker.Enqueue(req.Eval) - } - + n.handleUpsertedEval(req.Eval) return nil } @@ -867,10 +873,7 @@ func (n *nomadFSM) applyDeploymentAllocHealth(buf []byte, index uint64) interfac return err } - if req.Eval != nil && req.Eval.ShouldEnqueue() { - n.evalBroker.Enqueue(req.Eval) - } - + n.handleUpsertedEval(req.Eval) return nil } diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index b45be1b462f5..6bd564998441 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -476,7 +476,7 @@ func (n *Node) UpdateDrain(args *structs.NodeUpdateDrainRequest, } reply.NodeModifyIndex = index - // If the node is transistioning to be eligible, create Node evaluations + // If the node is transitioning to be eligible, create Node evaluations // because there may be a System job registered that should be evaluated. if node.SchedulingEligibility == structs.NodeSchedulingIneligible && args.MarkEligible && args.DrainStrategy == nil { evalIDs, evalIndex, err := n.createNodeEvals(args.NodeID, index) @@ -556,7 +556,7 @@ func (n *Node) UpdateEligibility(args *structs.NodeUpdateEligibilityRequest, } } - // If the node is transistioning to be eligible, create Node evaluations + // If the node is transitioning to be eligible, create Node evaluations // because there may be a System job registered that should be evaluated. if node.SchedulingEligibility == structs.NodeSchedulingIneligible && args.Eligibility == structs.NodeSchedulingEligible { evalIDs, evalIndex, err := n.createNodeEvals(args.NodeID, index) diff --git a/nomad/server.go b/nomad/server.go index 235e2988a3df..f6fbea91de60 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -893,7 +893,7 @@ func (s *Server) setupDeploymentWatcher() error { s.deploymentWatcher = deploymentwatcher.NewDeploymentsWatcher( s.logger, raftShim, deploymentwatcher.LimitStateQueriesPerSecond, - deploymentwatcher.CrossDeploymentEvalBatchDuration) + deploymentwatcher.CrossDeploymentUpdateBatchDuration) return nil } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index d92073336849..568fcbed6168 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -1890,7 +1890,13 @@ func (s *StateStore) nestedUpdateAllocFromClient(txn *memdb.Txn, index uint64, a copyAlloc.ClientStatus = alloc.ClientStatus copyAlloc.ClientDescription = alloc.ClientDescription copyAlloc.TaskStates = alloc.TaskStates + + // Merge the deployment status taking only what the client should set + oldDeploymentStatus := copyAlloc.DeploymentStatus copyAlloc.DeploymentStatus = alloc.DeploymentStatus + if oldDeploymentStatus != nil && oldDeploymentStatus.Canary { + copyAlloc.DeploymentStatus.Canary = true + } // Update the modify index copyAlloc.ModifyIndex = index @@ -1961,6 +1967,9 @@ func (s *StateStore) upsertAllocsImpl(index uint64, allocs []*structs.Allocation alloc.CreateIndex = index alloc.ModifyIndex = index alloc.AllocModifyIndex = index + if alloc.DeploymentStatus != nil { + alloc.DeploymentStatus.ModifyIndex = index + } // Issue https://github.com/hashicorp/nomad/issues/2583 uncovered // the a race between a forced garbage collection and the scheduler @@ -2085,6 +2094,12 @@ func (s *StateStore) UpdateAllocsDesiredTransitions(index uint64, allocs map[str } } + for _, eval := range evals { + if err := s.nestedUpsertEval(txn, index, eval); err != nil { + return err + } + } + // Update the indexes if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { return fmt.Errorf("index update failed: %v", err) @@ -2614,11 +2629,13 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD return err } + // groupIndex is a map of groups being promoted groupIndex := make(map[string]struct{}, len(req.Groups)) for _, g := range req.Groups { groupIndex[g] = struct{}{} } + // canaryIndex is the set of placed canaries in the deployment canaryIndex := make(map[string]struct{}, len(deployment.TaskGroups)) for _, state := range deployment.TaskGroups { for _, c := range state.PlacedCanaries { @@ -2626,8 +2643,13 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD } } - haveCanaries := false - var unhealthyErr multierror.Error + // healthyCounts is a mapping of group to the number of healthy canaries + healthyCounts := make(map[string]int, len(deployment.TaskGroups)) + + // promotable is the set of allocations that we can move from canary to + // non-canary + var promotable []*structs.Allocation + for { raw := iter.Next() if raw == nil { @@ -2648,19 +2670,32 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD // Ensure the canaries are healthy if !alloc.DeploymentStatus.IsHealthy() { - multierror.Append(&unhealthyErr, fmt.Errorf("Canary allocation %q for group %q is not healthy", alloc.ID, alloc.TaskGroup)) continue } - haveCanaries = true + healthyCounts[alloc.TaskGroup]++ + promotable = append(promotable, alloc) } - if err := unhealthyErr.ErrorOrNil(); err != nil { - return err + // Determine if we have enough healthy allocations + var unhealthyErr multierror.Error + for tg, state := range deployment.TaskGroups { + if _, ok := groupIndex[tg]; !req.All && !ok { + continue + } + + need := state.DesiredCanaries + if need == 0 { + continue + } + + if have := healthyCounts[tg]; have < need { + multierror.Append(&unhealthyErr, fmt.Errorf("Task group %q has %d/%d healthy allocations", tg, have, need)) + } } - if !haveCanaries { - return fmt.Errorf("no canaries to promote") + if err := unhealthyErr.ErrorOrNil(); err != nil { + return err } // Update deployment @@ -2692,6 +2727,24 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD } } + // For each promotable allocation remoce the canary field + for _, alloc := range promotable { + promoted := alloc.Copy() + promoted.DeploymentStatus.Canary = false + promoted.DeploymentStatus.ModifyIndex = index + promoted.ModifyIndex = index + promoted.AllocModifyIndex = index + + if err := txn.Insert("allocs", promoted); err != nil { + return fmt.Errorf("alloc insert failed: %v", err) + } + } + + // Update the alloc index + if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + txn.Commit() return nil } @@ -2715,7 +2768,7 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl // Update the health status of each allocation if total := len(req.HealthyAllocationIDs) + len(req.UnhealthyAllocationIDs); total != 0 { - setAllocHealth := func(id string, healthy bool) error { + setAllocHealth := func(id string, healthy bool, ts time.Time) error { existing, err := txn.First("allocs", "id", id) if err != nil { return fmt.Errorf("alloc %q lookup failed: %v", id, err) @@ -2735,6 +2788,7 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl copy.DeploymentStatus = &structs.AllocDeploymentStatus{} } copy.DeploymentStatus.Healthy = helper.BoolToPtr(healthy) + copy.DeploymentStatus.Timestamp = ts copy.DeploymentStatus.ModifyIndex = index if err := s.updateDeploymentWithAlloc(index, copy, old, txn); err != nil { @@ -2749,12 +2803,12 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl } for _, id := range req.HealthyAllocationIDs { - if err := setAllocHealth(id, true); err != nil { + if err := setAllocHealth(id, true, req.Timestamp); err != nil { return err } } for _, id := range req.UnhealthyAllocationIDs { - if err := setAllocHealth(id, false); err != nil { + if err := setAllocHealth(id, false, req.Timestamp); err != nil { return err } } @@ -3284,6 +3338,20 @@ func (s *StateStore) updateDeploymentWithAlloc(index uint64, alloc, existing *st state.HealthyAllocs += healthy state.UnhealthyAllocs += unhealthy + // Update the progress deadline + if pd := state.ProgressDeadline; pd != 0 { + // If we are the first placed allocation for the deployment start the progress deadline. + if placed != 0 && state.RequireProgressBy.IsZero() { + // Use modify time instead of create time because we may in-place + // update the allocation to be part of a new deployment. + state.RequireProgressBy = time.Unix(0, alloc.ModifyTime).Add(pd) + } else if healthy != 0 { + if d := alloc.DeploymentStatus.Timestamp.Add(pd); d.After(state.RequireProgressBy) { + state.RequireProgressBy = d + } + } + } + // Upsert the deployment if err := s.upsertDeploymentImpl(index, deploymentCopy, txn); err != nil { return err diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index cf0841e46409..2c9f42ff7fd5 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -3536,6 +3536,86 @@ func TestStateStore_UpdateMultipleAllocsFromClient(t *testing.T) { } } +func TestStateStore_UpdateAllocsFromClient_Deployment(t *testing.T) { + require := require.New(t) + state := testStateStore(t) + + alloc := mock.Alloc() + now := time.Now() + alloc.CreateTime = now.UnixNano() + pdeadline := 5 * time.Minute + deployment := mock.Deployment() + deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline + alloc.DeploymentID = deployment.ID + + require.Nil(state.UpsertJob(999, alloc.Job)) + require.Nil(state.UpsertDeployment(1000, deployment)) + require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc})) + + healthy := now.Add(time.Second) + update := &structs.Allocation{ + ID: alloc.ID, + ClientStatus: structs.AllocClientStatusRunning, + JobID: alloc.JobID, + TaskGroup: alloc.TaskGroup, + DeploymentStatus: &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + Timestamp: healthy, + }, + } + require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update})) + + // Check that the deployment state was updated because the healthy + // deployment + dout, err := state.DeploymentByID(nil, deployment.ID) + require.Nil(err) + require.NotNil(dout) + require.Len(dout.TaskGroups, 1) + dstate := dout.TaskGroups[alloc.TaskGroup] + require.NotNil(dstate) + require.Equal(1, dstate.PlacedAllocs) + require.True(healthy.Add(pdeadline).Equal(dstate.RequireProgressBy)) +} + +// This tests that the deployment state is merged correctly +func TestStateStore_UpdateAllocsFromClient_DeploymentStateMerges(t *testing.T) { + require := require.New(t) + state := testStateStore(t) + + alloc := mock.Alloc() + now := time.Now() + alloc.CreateTime = now.UnixNano() + pdeadline := 5 * time.Minute + deployment := mock.Deployment() + deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline + alloc.DeploymentID = deployment.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + } + + require.Nil(state.UpsertJob(999, alloc.Job)) + require.Nil(state.UpsertDeployment(1000, deployment)) + require.Nil(state.UpsertAllocs(1001, []*structs.Allocation{alloc})) + + update := &structs.Allocation{ + ID: alloc.ID, + ClientStatus: structs.AllocClientStatusRunning, + JobID: alloc.JobID, + TaskGroup: alloc.TaskGroup, + DeploymentStatus: &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(true), + Canary: false, + }, + } + require.Nil(state.UpdateAllocsFromClient(1001, []*structs.Allocation{update})) + + // Check that the merging of the deployment status was correct + out, err := state.AllocByID(nil, alloc.ID) + require.Nil(err) + require.NotNil(out) + require.True(out.DeploymentStatus.Canary) +} + func TestStateStore_UpsertAlloc_Alloc(t *testing.T) { state := testStateStore(t) alloc := mock.Alloc() @@ -3610,28 +3690,26 @@ func TestStateStore_UpsertAlloc_Alloc(t *testing.T) { } func TestStateStore_UpsertAlloc_Deployment(t *testing.T) { + require := require.New(t) state := testStateStore(t) - deployment := mock.Deployment() alloc := mock.Alloc() + now := time.Now() + alloc.CreateTime = now.UnixNano() + alloc.ModifyTime = now.UnixNano() + pdeadline := 5 * time.Minute + deployment := mock.Deployment() + deployment.TaskGroups[alloc.TaskGroup].ProgressDeadline = pdeadline alloc.DeploymentID = deployment.ID - if err := state.UpsertJob(999, alloc.Job); err != nil { - t.Fatalf("err: %v", err) - } - if err := state.UpsertDeployment(1000, deployment); err != nil { - t.Fatalf("err: %v", err) - } + require.Nil(state.UpsertJob(999, alloc.Job)) + require.Nil(state.UpsertDeployment(1000, deployment)) // Create a watch set so we can test that update fires the watch ws := memdb.NewWatchSet() - if _, err := state.AllocsByDeployment(ws, alloc.DeploymentID); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.AllocsByDeployment(ws, alloc.DeploymentID)) err := state.UpsertAllocs(1001, []*structs.Allocation{alloc}) - if err != nil { - t.Fatalf("err: %v", err) - } + require.Nil(err) if !watchFired(ws) { t.Fatalf("watch not fired") @@ -3639,29 +3717,26 @@ func TestStateStore_UpsertAlloc_Deployment(t *testing.T) { ws = memdb.NewWatchSet() allocs, err := state.AllocsByDeployment(ws, alloc.DeploymentID) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(allocs) != 1 { - t.Fatalf("bad: %#v", allocs) - } - - if !reflect.DeepEqual(alloc, allocs[0]) { - t.Fatalf("bad: %#v %#v", alloc, allocs[0]) - } + require.Nil(err) + require.Len(allocs, 1) + require.EqualValues(alloc, allocs[0]) index, err := state.Index("allocs") - if err != nil { - t.Fatalf("err: %v", err) - } - if index != 1001 { - t.Fatalf("bad: %d", index) - } - + require.Nil(err) + require.EqualValues(1001, index) if watchFired(ws) { t.Fatalf("bad") } + + // Check that the deployment state was updated + dout, err := state.DeploymentByID(nil, deployment.ID) + require.Nil(err) + require.NotNil(dout) + require.Len(dout.TaskGroups, 1) + dstate := dout.TaskGroups[alloc.TaskGroup] + require.NotNil(dstate) + require.Equal(1, dstate.PlacedAllocs) + require.True(now.Add(pdeadline).Equal(dstate.RequireProgressBy)) } // Testing to ensure we keep issue @@ -3981,6 +4056,11 @@ func TestStateStore_UpdateAllocDesiredTransition(t *testing.T) { require.Nil(err) require.EqualValues(1001, index) + // Check the eval is created + eout, err := state.EvalByID(nil, eval.ID) + require.Nil(err) + require.NotNil(eout) + m = map[string]*structs.DesiredTransition{alloc.ID: t2} require.Nil(state.UpdateAllocsDesiredTransitions(1002, m, evals)) @@ -5494,19 +5574,17 @@ func TestStateStore_UpsertDeploymentPromotion_Terminal(t *testing.T) { // Test promoting unhealthy canaries in a deployment. func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) { state := testStateStore(t) + require := require.New(t) // Create a job j := mock.Job() - if err := state.UpsertJob(1, j); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpsertJob(1, j)) // Create a deployment d := mock.Deployment() d.JobID = j.ID - if err := state.UpsertDeployment(2, d); err != nil { - t.Fatalf("bad: %v", err) - } + d.TaskGroups["web"].DesiredCanaries = 2 + require.Nil(state.UpsertDeployment(2, d)) // Create a set of allocations c1 := mock.Alloc() @@ -5518,9 +5596,7 @@ func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) { c2.DeploymentID = d.ID d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID) - if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil { - t.Fatalf("err: %v", err) - } + require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2})) // Promote the canaries req := &structs.ApplyDeploymentPromoteRequest{ @@ -5530,33 +5606,24 @@ func TestStateStore_UpsertDeploymentPromotion_Unhealthy(t *testing.T) { }, } err := state.UpdateDeploymentPromotion(4, req) - if err == nil { - t.Fatalf("bad: %v", err) - } - if !strings.Contains(err.Error(), c1.ID) { - t.Fatalf("expect canary %q to be listed as unhealth: %v", c1.ID, err) - } - if !strings.Contains(err.Error(), c2.ID) { - t.Fatalf("expect canary %q to be listed as unhealth: %v", c2.ID, err) - } + require.NotNil(err) + require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`) } // Test promoting a deployment with no canaries func TestStateStore_UpsertDeploymentPromotion_NoCanaries(t *testing.T) { state := testStateStore(t) + require := require.New(t) // Create a job j := mock.Job() - if err := state.UpsertJob(1, j); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpsertJob(1, j)) // Create a deployment d := mock.Deployment() + d.TaskGroups["web"].DesiredCanaries = 2 d.JobID = j.ID - if err := state.UpsertDeployment(2, d); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpsertDeployment(2, d)) // Promote the canaries req := &structs.ApplyDeploymentPromoteRequest{ @@ -5566,12 +5633,8 @@ func TestStateStore_UpsertDeploymentPromotion_NoCanaries(t *testing.T) { }, } err := state.UpdateDeploymentPromotion(4, req) - if err == nil { - t.Fatalf("bad: %v", err) - } - if !strings.Contains(err.Error(), "no canaries to promote") { - t.Fatalf("expect error promoting nonexistent canaries: %v", err) - } + require.NotNil(err) + require.Contains(err.Error(), `Task group "web" has 0/2 healthy allocations`) } // Test promoting all canaries in a deployment. @@ -5674,6 +5737,7 @@ func TestStateStore_UpsertDeploymentPromotion_All(t *testing.T) { // Test promoting a subset of canaries in a deployment. func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { state := testStateStore(t) + require := require.New(t) // Create a job with two task groups j := mock.Job() @@ -5681,9 +5745,7 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { tg2 := tg1.Copy() tg2.Name = "foo" j.TaskGroups = append(j.TaskGroups, tg2) - if err := state.UpsertJob(1, j); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpsertJob(1, j)) // Create a deployment d := mock.Deployment() @@ -5698,18 +5760,19 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { DesiredCanaries: 1, }, } - if err := state.UpsertDeployment(2, d); err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpsertDeployment(2, d)) - // Create a set of allocations + // Create a set of allocations for both groups, including an unhealthy one c1 := mock.Alloc() c1.JobID = j.ID c1.DeploymentID = d.ID d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID) c1.DeploymentStatus = &structs.AllocDeploymentStatus{ Healthy: helper.BoolToPtr(true), + Canary: true, } + + // Should still be a canary c2 := mock.Alloc() c2.JobID = j.ID c2.DeploymentID = d.ID @@ -5717,12 +5780,20 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { c2.TaskGroup = tg2.Name c2.DeploymentStatus = &structs.AllocDeploymentStatus{ Healthy: helper.BoolToPtr(true), + Canary: true, } - if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil { - t.Fatalf("err: %v", err) + c3 := mock.Alloc() + c3.JobID = j.ID + c3.DeploymentID = d.ID + d.TaskGroups[c3.TaskGroup].PlacedCanaries = append(d.TaskGroups[c3.TaskGroup].PlacedCanaries, c3.ID) + c3.DeploymentStatus = &structs.AllocDeploymentStatus{ + Healthy: helper.BoolToPtr(false), + Canary: true, } + require.Nil(state.UpsertAllocs(3, []*structs.Allocation{c1, c2, c3})) + // Create an eval e := mock.Eval() @@ -5734,36 +5805,34 @@ func TestStateStore_UpsertDeploymentPromotion_Subset(t *testing.T) { }, Eval: e, } - err := state.UpdateDeploymentPromotion(4, req) - if err != nil { - t.Fatalf("bad: %v", err) - } + require.Nil(state.UpdateDeploymentPromotion(4, req)) // Check that the status per task group was updated properly ws := memdb.NewWatchSet() dout, err := state.DeploymentByID(ws, d.ID) - if err != nil { - t.Fatalf("bad: %v", err) - } - if len(dout.TaskGroups) != 2 { - t.Fatalf("bad: %#v", dout.TaskGroups) - } - stateout, ok := dout.TaskGroups["web"] - if !ok { - t.Fatalf("bad: no state for task group web") - } - if !stateout.Promoted { - t.Fatalf("bad: task group web not promoted: %#v", stateout) - } + require.Nil(err) + require.Len(dout.TaskGroups, 2) + require.Contains(dout.TaskGroups, "web") + require.True(dout.TaskGroups["web"].Promoted) // Check that the evaluation was created - eout, _ := state.EvalByID(ws, e.ID) - if err != nil { - t.Fatalf("bad: %v", err) - } - if eout == nil { - t.Fatalf("bad: %#v", eout) - } + eout, err := state.EvalByID(ws, e.ID) + require.Nil(err) + require.NotNil(eout) + + // Check the canary field was set properly + aout1, err1 := state.AllocByID(ws, c1.ID) + aout2, err2 := state.AllocByID(ws, c2.ID) + aout3, err3 := state.AllocByID(ws, c3.ID) + require.Nil(err1) + require.Nil(err2) + require.Nil(err3) + require.NotNil(aout1) + require.NotNil(aout2) + require.NotNil(aout3) + require.False(aout1.DeploymentStatus.Canary) + require.True(aout2.DeploymentStatus.Canary) + require.True(aout3.DeploymentStatus.Canary) } // Test that allocation health can't be set against a nonexistent deployment @@ -5872,6 +5941,7 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { // Insert a deployment d := mock.Deployment() + d.TaskGroups["web"].ProgressDeadline = 5 * time.Minute if err := state.UpsertDeployment(1, d); err != nil { t.Fatalf("bad: %v", err) } @@ -5899,6 +5969,9 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { StatusDescription: desc, } + // Capture the time for the update + ts := time.Now() + // Set health against the deployment req := &structs.ApplyDeploymentAllocHealthRequest{ DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ @@ -5909,6 +5982,7 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { Job: j, Eval: e, DeploymentUpdate: u, + Timestamp: ts, } err := state.UpdateDeploymentAllocHealth(3, req) if err != nil { @@ -5959,6 +6033,13 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { if !out2.DeploymentStatus.IsUnhealthy() { t.Fatalf("bad: alloc %q not unhealthy", out2.ID) } + + if !out1.DeploymentStatus.Timestamp.Equal(ts) { + t.Fatalf("bad: alloc %q had timestamp %v; want %v", out1.ID, out1.DeploymentStatus.Timestamp, ts) + } + if !out2.DeploymentStatus.Timestamp.Equal(ts) { + t.Fatalf("bad: alloc %q had timestamp %v; want %v", out2.ID, out2.DeploymentStatus.Timestamp, ts) + } } func TestStateStore_UpsertVaultAccessors(t *testing.T) { diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index bd7e8727b355..f6bef46695cf 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -533,6 +533,15 @@ func serviceDiff(old, new *Service, contextual bool) *ObjectDiff { // Diff the primitive fields. diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + if setDiff := stringSetDiff(old.CanaryTags, new.CanaryTags, "CanaryTags", contextual); setDiff != nil { + diff.Objects = append(diff.Objects, setDiff) + } + + // Tag diffs + if setDiff := stringSetDiff(old.Tags, new.Tags, "Tags", contextual); setDiff != nil { + diff.Objects = append(diff.Objects, setDiff) + } + // Checks diffs if cDiffs := serviceCheckDiffs(old.Checks, new.Checks, contextual); cDiffs != nil { diff.Objects = append(diff.Objects, cDiffs...) diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index f3b3557f984b..879522ecdb69 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -1787,6 +1787,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "0", New: "", }, + { + Type: DiffTypeDeleted, + Name: "ProgressDeadline", + Old: "0", + New: "", + }, }, }, }, @@ -1837,6 +1843,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "", New: "0", }, + { + Type: DiffTypeAdded, + Name: "ProgressDeadline", + Old: "", + New: "0", + }, }, }, }, @@ -1846,22 +1858,24 @@ func TestTaskGroupDiff(t *testing.T) { // Update strategy edited Old: &TaskGroup{ Update: &UpdateStrategy{ - MaxParallel: 5, - HealthCheck: "foo", - MinHealthyTime: 1 * time.Second, - HealthyDeadline: 30 * time.Second, - AutoRevert: true, - Canary: 2, + MaxParallel: 5, + HealthCheck: "foo", + MinHealthyTime: 1 * time.Second, + HealthyDeadline: 30 * time.Second, + ProgressDeadline: 29 * time.Second, + AutoRevert: true, + Canary: 2, }, }, New: &TaskGroup{ Update: &UpdateStrategy{ - MaxParallel: 7, - HealthCheck: "bar", - MinHealthyTime: 2 * time.Second, - HealthyDeadline: 31 * time.Second, - AutoRevert: false, - Canary: 1, + MaxParallel: 7, + HealthCheck: "bar", + MinHealthyTime: 2 * time.Second, + HealthyDeadline: 31 * time.Second, + ProgressDeadline: 32 * time.Second, + AutoRevert: false, + Canary: 1, }, }, Expected: &TaskGroupDiff{ @@ -1907,6 +1921,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "1000000000", New: "2000000000", }, + { + Type: DiffTypeEdited, + Name: "ProgressDeadline", + Old: "29000000000", + New: "32000000000", + }, }, }, }, @@ -1917,22 +1937,24 @@ func TestTaskGroupDiff(t *testing.T) { Contextual: true, Old: &TaskGroup{ Update: &UpdateStrategy{ - MaxParallel: 5, - HealthCheck: "foo", - MinHealthyTime: 1 * time.Second, - HealthyDeadline: 30 * time.Second, - AutoRevert: true, - Canary: 2, + MaxParallel: 5, + HealthCheck: "foo", + MinHealthyTime: 1 * time.Second, + HealthyDeadline: 30 * time.Second, + ProgressDeadline: 30 * time.Second, + AutoRevert: true, + Canary: 2, }, }, New: &TaskGroup{ Update: &UpdateStrategy{ - MaxParallel: 7, - HealthCheck: "foo", - MinHealthyTime: 1 * time.Second, - HealthyDeadline: 30 * time.Second, - AutoRevert: true, - Canary: 2, + MaxParallel: 7, + HealthCheck: "foo", + MinHealthyTime: 1 * time.Second, + HealthyDeadline: 30 * time.Second, + ProgressDeadline: 30 * time.Second, + AutoRevert: true, + Canary: 2, }, }, Expected: &TaskGroupDiff{ @@ -1978,6 +2000,12 @@ func TestTaskGroupDiff(t *testing.T) { Old: "1000000000", New: "1000000000", }, + { + Type: DiffTypeNone, + Name: "ProgressDeadline", + Old: "30000000000", + New: "30000000000", + }, }, }, }, @@ -3428,6 +3456,99 @@ func TestTaskDiff(t *testing.T) { }, }, }, + { + Name: "Services tags edited (no checks) with context", + Contextual: true, + Old: &Task{ + Services: []*Service{ + { + Tags: []string{"foo", "bar"}, + CanaryTags: []string{"foo", "bar"}, + }, + }, + }, + New: &Task{ + Services: []*Service{ + { + Tags: []string{"bar", "bam"}, + CanaryTags: []string{"bar", "bam"}, + }, + }, + }, + Expected: &TaskDiff{ + Type: DiffTypeEdited, + Objects: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "Service", + Objects: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "CanaryTags", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "CanaryTags", + Old: "", + New: "bam", + }, + { + Type: DiffTypeNone, + Name: "CanaryTags", + Old: "bar", + New: "bar", + }, + { + Type: DiffTypeDeleted, + Name: "CanaryTags", + Old: "foo", + New: "", + }, + }, + }, + { + Type: DiffTypeEdited, + Name: "Tags", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "Tags", + Old: "", + New: "bam", + }, + { + Type: DiffTypeNone, + Name: "Tags", + Old: "bar", + New: "bar", + }, + { + Type: DiffTypeDeleted, + Name: "Tags", + Old: "foo", + New: "", + }, + }, + }, + }, + Fields: []*FieldDiff{ + { + Type: DiffTypeNone, + Name: "AddressMode", + }, + { + Type: DiffTypeNone, + Name: "Name", + }, + { + Type: DiffTypeNone, + Name: "PortLabel", + }, + }, + }, + }, + }, + }, { Name: "Service Checks edited", Old: &Task{ diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 40a9a34d37e6..70fbc05546f9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -788,6 +788,9 @@ type DeploymentAllocHealthRequest struct { type ApplyDeploymentAllocHealthRequest struct { DeploymentAllocHealthRequest + // Timestamp is the timestamp to use when setting the allocations health. + Timestamp time.Time + // An optional field to update the status of a deployment DeploymentUpdate *DeploymentStatusUpdate @@ -2478,13 +2481,14 @@ var ( // DefaultUpdateStrategy provides a baseline that can be used to upgrade // jobs with the old policy or for populating field defaults. DefaultUpdateStrategy = &UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 1, - HealthCheck: UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 5 * time.Minute, - AutoRevert: false, - Canary: 0, + Stagger: 30 * time.Second, + MaxParallel: 1, + HealthCheck: UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 5 * time.Minute, + ProgressDeadline: 10 * time.Minute, + AutoRevert: false, + Canary: 0, } ) @@ -2511,6 +2515,12 @@ type UpdateStrategy struct { // period doesn't count against the MinHealthyTime. HealthyDeadline time.Duration + // ProgressDeadline is the time in which an allocation as part of the + // deployment must transition to healthy. If no allocation becomes healthy + // after the deadline, the deployment is marked as failed. If the deadline + // is zero, the first failure causes the deployment to fail. + ProgressDeadline time.Duration + // AutoRevert declares that if a deployment fails because of unhealthy // allocations, there should be an attempt to auto-revert the job to a // stable version. @@ -2555,9 +2565,15 @@ func (u *UpdateStrategy) Validate() error { if u.HealthyDeadline <= 0 { multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be greater than zero: %v", u.HealthyDeadline)) } + if u.ProgressDeadline < 0 { + multierror.Append(&mErr, fmt.Errorf("Progress deadline must be zero or greater: %v", u.ProgressDeadline)) + } if u.MinHealthyTime >= u.HealthyDeadline { multierror.Append(&mErr, fmt.Errorf("Minimum healthy time must be less than healthy deadline: %v > %v", u.MinHealthyTime, u.HealthyDeadline)) } + if u.ProgressDeadline != 0 && u.HealthyDeadline >= u.ProgressDeadline { + multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be less than progress deadline: %v > %v", u.HealthyDeadline, u.ProgressDeadline)) + } if u.Stagger <= 0 { multierror.Append(&mErr, fmt.Errorf("Stagger must be greater than zero: %v", u.Stagger)) } @@ -3734,8 +3750,9 @@ type Service struct { // this service. AddressMode string - Tags []string // List of tags for the service - Checks []*ServiceCheck // List of checks associated with the service + Tags []string // List of tags for the service + CanaryTags []string // List of tags for the service when it is a canary + Checks []*ServiceCheck // List of checks associated with the service } func (s *Service) Copy() *Service { @@ -3745,6 +3762,7 @@ func (s *Service) Copy() *Service { ns := new(Service) *ns = *s ns.Tags = helper.CopySliceString(ns.Tags) + ns.CanaryTags = helper.CopySliceString(ns.CanaryTags) if s.Checks != nil { checks := make([]*ServiceCheck, len(ns.Checks)) @@ -3765,6 +3783,9 @@ func (s *Service) Canonicalize(job string, taskGroup string, task string) { if len(s.Tags) == 0 { s.Tags = nil } + if len(s.CanaryTags) == 0 { + s.CanaryTags = nil + } if len(s.Checks) == 0 { s.Checks = nil } @@ -3832,7 +3853,7 @@ func (s *Service) ValidateName(name string) error { // Hash returns a base32 encoded hash of a Service's contents excluding checks // as they're hashed independently. -func (s *Service) Hash(allocID, taskName string) string { +func (s *Service) Hash(allocID, taskName string, canary bool) string { h := sha1.New() io.WriteString(h, allocID) io.WriteString(h, taskName) @@ -3842,6 +3863,14 @@ func (s *Service) Hash(allocID, taskName string) string { for _, tag := range s.Tags { io.WriteString(h, tag) } + for _, tag := range s.CanaryTags { + io.WriteString(h, tag) + } + + // Vary ID on whether or not CanaryTags will be used + if canary { + h.Write([]byte("Canary")) + } // Base32 is used for encoding the hash as sha1 hashes can always be // encoded without padding, only 4 bytes larger than base64, and saves @@ -5300,6 +5329,7 @@ const ( DeploymentStatusDescriptionStoppedJob = "Cancelled because job is stopped" DeploymentStatusDescriptionNewerJob = "Cancelled due to newer version of job" DeploymentStatusDescriptionFailedAllocations = "Failed due to unhealthy allocations" + DeploymentStatusDescriptionProgressDeadline = "Failed due to progress deadline" DeploymentStatusDescriptionFailedByUser = "Deployment marked as failed" ) @@ -5452,6 +5482,14 @@ type DeploymentState struct { // reverted on failure AutoRevert bool + // ProgressDeadline is the deadline by which an allocation must transition + // to healthy before the deployment is considered failed. + ProgressDeadline time.Duration + + // RequireProgressBy is the time by which an allocation must transition + // to healthy before the deployment is considered failed. + RequireProgressBy time.Time + // Promoted marks whether the canaries have been promoted Promoted bool @@ -5563,6 +5601,13 @@ type DesiredTransition struct { // Migrate is used to indicate that this allocation should be stopped and // migrated to another node. Migrate *bool + + // Reschedule is used to indicate that this allocation is eligible to be + // rescheduled. Most allocations are automatically eligible for + // rescheduling, so this field is only required when an allocation is not + // automatically eligible. An example is an allocation that is part of a + // deployment. + Reschedule *bool } // Merge merges the two desired transitions, preferring the values from the @@ -5571,6 +5616,10 @@ func (d *DesiredTransition) Merge(o *DesiredTransition) { if o.Migrate != nil { d.Migrate = o.Migrate } + + if o.Reschedule != nil { + d.Reschedule = o.Reschedule + } } // ShouldMigrate returns whether the transition object dictates a migration. @@ -5578,6 +5627,12 @@ func (d *DesiredTransition) ShouldMigrate() bool { return d.Migrate != nil && *d.Migrate } +// ShouldReschedule returns whether the transition object dictates a +// rescheduling. +func (d *DesiredTransition) ShouldReschedule() bool { + return d.Reschedule != nil && *d.Reschedule +} + const ( AllocDesiredStatusRun = "run" // Allocation should run AllocDesiredStatusStop = "stop" // Allocation should stop @@ -5997,6 +6052,7 @@ func (a *Allocation) Stub() *AllocListStub { DesiredDescription: a.DesiredDescription, ClientStatus: a.ClientStatus, ClientDescription: a.ClientDescription, + DesiredTransition: a.DesiredTransition, TaskStates: a.TaskStates, DeploymentStatus: a.DeploymentStatus, FollowupEvalID: a.FollowupEvalID, @@ -6021,6 +6077,7 @@ type AllocListStub struct { DesiredDescription string ClientStatus string ClientDescription string + DesiredTransition DesiredTransition TaskStates map[string]*TaskState DeploymentStatus *AllocDeploymentStatus FollowupEvalID string @@ -6172,6 +6229,13 @@ type AllocDeploymentStatus struct { // healthy or unhealthy. Healthy *bool + // Timestamp is the time at which the health status was set. + Timestamp time.Time + + // Canary marks whether the allocation is a canary or not. A canary that has + // been promoted will have this field set to false. + Canary bool + // ModifyIndex is the raft index in which the deployment status was last // changed. ModifyIndex uint64 @@ -6202,6 +6266,15 @@ func (a *AllocDeploymentStatus) IsUnhealthy() bool { return a.Healthy != nil && !*a.Healthy } +// IsCanary returns if the allocation is marked as a canary +func (a *AllocDeploymentStatus) IsCanary() bool { + if a == nil { + return false + } + + return a.Canary +} + func (a *AllocDeploymentStatus) Copy() *AllocDeploymentStatus { if a == nil { return nil diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index ec65fe755af2..391a73f13af5 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -1565,12 +1565,13 @@ func TestConstraint_Validate(t *testing.T) { func TestUpdateStrategy_Validate(t *testing.T) { u := &UpdateStrategy{ - MaxParallel: 0, - HealthCheck: "foo", - MinHealthyTime: -10, - HealthyDeadline: -15, - AutoRevert: false, - Canary: -1, + MaxParallel: 0, + HealthCheck: "foo", + MinHealthyTime: -10, + HealthyDeadline: -15, + ProgressDeadline: -25, + AutoRevert: false, + Canary: -1, } err := u.Validate() @@ -1590,7 +1591,13 @@ func TestUpdateStrategy_Validate(t *testing.T) { if !strings.Contains(mErr.Errors[4].Error(), "Healthy deadline must be greater than zero") { t.Fatalf("err: %s", err) } - if !strings.Contains(mErr.Errors[5].Error(), "Minimum healthy time must be less than healthy deadline") { + if !strings.Contains(mErr.Errors[5].Error(), "Progress deadline must be zero or greater") { + t.Fatalf("err: %s", err) + } + if !strings.Contains(mErr.Errors[6].Error(), "Minimum healthy time must be less than healthy deadline") { + t.Fatalf("err: %s", err) + } + if !strings.Contains(mErr.Errors[7].Error(), "Healthy deadline must be less than progress deadline") { t.Fatalf("err: %s", err) } } diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index b9bdb8b17ef0..e0ad0074cd92 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -499,11 +499,15 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul } // If we are placing a canary and we found a match, add the canary - // to the deployment state object. + // to the deployment state object and mark it as a canary. if missing.Canary() { if state, ok := s.deployment.TaskGroups[tg.Name]; ok { state.PlacedCanaries = append(state.PlacedCanaries, alloc.ID) } + + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + } } // Track the placement diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 43308913a11d..0618b5c720e8 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -1814,6 +1814,11 @@ func TestServiceSched_JobModify_Canaries(t *testing.T) { if len(planned) != desiredUpdates { t.Fatalf("bad: %#v", plan) } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } h.AssertEvalStatus(t, structs.EvalStatusComplete) @@ -3108,67 +3113,92 @@ func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { } -// Tests that deployments with failed allocs don't result in placements -func TestDeployment_FailedAllocs_NoReschedule(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and a reschedule policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) - - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + // Generate a fake job with allocations and a reschedule policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + } + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) + + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed + } - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = deployment.ID + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + }) } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) - - // Verify no plan created - require.Equal(0, len(h.Plans)) - } func TestBatchSched_Run_CompleteAlloc(t *testing.T) { diff --git a/scheduler/reconcile.go b/scheduler/reconcile.go index 485bfa55ebf2..63c9cff7286c 100644 --- a/scheduler/reconcile.go +++ b/scheduler/reconcile.go @@ -194,20 +194,8 @@ func (a *allocReconciler) Compute() *reconcileResults { // Detect if the deployment is paused if a.deployment != nil { - // Detect if any allocs associated with this deploy have failed - // Failed allocations could edge trigger an evaluation before the deployment watcher - // runs and marks the deploy as failed. This block makes sure that is still - // considered a failed deploy - failedAllocsInDeploy := false - for _, as := range m { - for _, alloc := range as { - if alloc.DeploymentID == a.deployment.ID && alloc.ClientStatus == structs.AllocClientStatusFailed { - failedAllocsInDeploy = true - } - } - } a.deploymentPaused = a.deployment.Status == structs.DeploymentStatusPaused - a.deploymentFailed = a.deployment.Status == structs.DeploymentStatusFailed || failedAllocsInDeploy + a.deploymentFailed = a.deployment.Status == structs.DeploymentStatusFailed } // Reconcile each group @@ -334,12 +322,10 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { dstate, existingDeployment = a.deployment.TaskGroups[group] } if !existingDeployment { - autorevert := false - if tg.Update != nil && tg.Update.AutoRevert { - autorevert = true - } - dstate = &structs.DeploymentState{ - AutoRevert: autorevert, + dstate = &structs.DeploymentState{} + if tg.Update != nil { + dstate.AutoRevert = tg.Update.AutoRevert + dstate.ProgressDeadline = tg.Update.ProgressDeadline } } @@ -348,13 +334,15 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { all, ignore := a.filterOldTerminalAllocs(all) desiredChanges.Ignore += uint64(len(ignore)) + // canaries is the set of canaries for the current deployment and all is all + // allocs including the canaries canaries, all := a.handleGroupCanaries(all, desiredChanges) // Determine what set of allocations are on tainted nodes untainted, migrate, lost := all.filterByTainted(a.taintedNodes) // Determine what set of terminal allocations need to be rescheduled - untainted, rescheduleNow, rescheduleLater := untainted.filterByRescheduleable(a.batch, a.now, a.evalID) + untainted, rescheduleNow, rescheduleLater := untainted.filterByRescheduleable(a.batch, a.now, a.evalID, a.deployment) // Create batched follow up evaluations for allocations that are // reschedulable later and mark the allocations for in place updating @@ -371,14 +359,6 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { desiredChanges.Stop += uint64(len(stop)) untainted = untainted.difference(stop) - // Having stopped un-needed allocations, append the canaries to the existing - // set of untainted because they are promoted. This will cause them to be - // treated like non-canaries - if !canaryState { - untainted = untainted.union(canaries) - nameIndex.Set(canaries) - } - // Do inplace upgrades where possible and capture the set of upgrades that // need to be done destructively. ignore, inplace, destructive := a.computeUpdates(tg, untainted) @@ -388,6 +368,12 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { dstate.DesiredTotal += len(destructive) + len(inplace) } + // Remove the canaries now that we have handled rescheduling so that we do + // not consider them when making placement decisions. + if canaryState { + untainted = untainted.difference(canaries) + } + // The fact that we have destructive updates and have less canaries than is // desired means we need to create canaries numDestructive := len(destructive) @@ -396,7 +382,6 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { requireCanary := numDestructive != 0 && strategy != nil && len(canaries) < strategy.Canary && !canariesPromoted if requireCanary && !a.deploymentPaused && !a.deploymentFailed { number := strategy.Canary - len(canaries) - number = helper.IntMin(numDestructive, number) desiredChanges.Canary += uint64(number) if !existingDeployment { dstate.DesiredCanaries = strategy.Canary @@ -436,16 +421,29 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { min := helper.IntMin(len(place), limit) limit -= min - } else if !deploymentPlaceReady && len(lost) != 0 { - // We are in a situation where we shouldn't be placing more than we need - // to but we have lost allocations. It is a very weird user experience - // if you have a node go down and Nomad doesn't replace the allocations - // because the deployment is paused/failed so we only place to recover - // the lost allocations. - allowed := helper.IntMin(len(lost), len(place)) - desiredChanges.Place += uint64(allowed) - for _, p := range place[:allowed] { - a.result.place = append(a.result.place, p) + } else if !deploymentPlaceReady { + // We do not want to place additional allocations but in the case we + // have lost allocations or allocations that require rescheduling now, + // we do so regardless to avoid odd user experiences. + if len(lost) != 0 { + allowed := helper.IntMin(len(lost), len(place)) + desiredChanges.Place += uint64(allowed) + for _, p := range place[:allowed] { + a.result.place = append(a.result.place, p) + } + } + + // Handle rescheduling of failed allocations even if the deployment is + // failed. We do not reschedule if the allocation is part of the failed + // deployment. + if now := len(rescheduleNow); now != 0 { + for _, p := range place { + prev := p.PreviousAllocation() + if p.IsRescheduling() && !(a.deploymentFailed && prev != nil && a.deployment.ID == prev.DeploymentID) { + a.result.place = append(a.result.place, p) + desiredChanges.Place++ + } + } } } @@ -522,16 +520,15 @@ func (a *allocReconciler) computeGroup(group string, all allocSet) bool { // deploymentComplete is whether the deployment is complete which largely // means that no placements were made or desired to be made - deploymentComplete := len(destructive)+len(inplace)+len(place)+len(migrate) == 0 && !requireCanary + deploymentComplete := len(destructive)+len(inplace)+len(place)+len(migrate)+len(rescheduleNow)+len(rescheduleLater) == 0 && !requireCanary // Final check to see if the deployment is complete is to ensure everything // is healthy if deploymentComplete && a.deployment != nil { - partOf, _ := untainted.filterByDeployment(a.deployment.ID) - for _, alloc := range partOf { - if !alloc.DeploymentStatus.IsHealthy() { + if dstate, ok := a.deployment.TaskGroups[group]; ok { + if dstate.HealthyAllocs < helper.IntMax(dstate.DesiredTotal, dstate.DesiredCanaries) || // Make sure we have enough healthy allocs + (dstate.DesiredCanaries > 0 && !dstate.Promoted) { // Make sure we are promoted if we have canaries deploymentComplete = false - break } } } @@ -663,26 +660,24 @@ func (a *allocReconciler) computeLimit(group *structs.TaskGroup, untainted, dest func (a *allocReconciler) computePlacements(group *structs.TaskGroup, nameIndex *allocNameIndex, untainted, migrate allocSet, reschedule allocSet) []allocPlaceResult { - // Hot path the nothing to do case - existing := len(untainted) + len(migrate) - if existing >= group.Count { - return nil - } - var place []allocPlaceResult // Add rescheduled placement results - // Any allocations being rescheduled will remain at DesiredStatusRun ClientStatusFailed + var place []allocPlaceResult for _, alloc := range reschedule { place = append(place, allocPlaceResult{ name: alloc.Name, taskGroup: group, previousAlloc: alloc, reschedule: true, + canary: alloc.DeploymentStatus.IsCanary(), }) - existing += 1 - if existing == group.Count { - break - } } + + // Hot path the nothing to do case + existing := len(untainted) + len(migrate) + len(reschedule) + if existing >= group.Count { + return place + } + // Add remaining placement results if existing < group.Count { for _, name := range nameIndex.Next(uint(group.Count - existing)) { diff --git a/scheduler/reconcile_test.go b/scheduler/reconcile_test.go index e24ccfaec1e6..9d22439e2ffb 100644 --- a/scheduler/reconcile_test.go +++ b/scheduler/reconcile_test.go @@ -19,69 +19,6 @@ import ( "github.com/stretchr/testify/require" ) -/* -Basic Tests: -√ Place when there is nothing in the cluster -√ Place remainder when there is some in the cluster -√ Scale down from n to n-m where n != m -√ Scale down from n to zero -√ Inplace upgrade test -√ Inplace upgrade and scale up test -√ Inplace upgrade and scale down test -√ Destructive upgrade -√ Destructive upgrade and scale up test -√ Destructive upgrade and scale down test -√ Handle lost nodes -√ Handle lost nodes and scale up -√ Handle lost nodes and scale down -√ Handle draining nodes -√ Handle draining nodes and scale up -√ Handle draining nodes and scale down -√ Handle task group being removed -√ Handle job being stopped both as .Stopped and nil -√ Place more that one group -√ Handle delayed rescheduling failed allocs for batch jobs -√ Handle delayed rescheduling failed allocs for service jobs -√ Handle eligible now rescheduling failed allocs for batch jobs -√ Handle eligible now rescheduling failed allocs for service jobs -√ Previously rescheduled allocs should not be rescheduled again -√ Aggregated evaluations for allocations that fail close together - -Update stanza Tests: -√ Stopped job cancels any active deployment -√ Stopped job doesn't cancel terminal deployment -√ JobIndex change cancels any active deployment -√ JobIndex change doesn't cancels any terminal deployment -√ Destructive changes create deployment and get rolled out via max_parallelism -√ Don't create a deployment if there are no changes -√ Deployment created by all inplace updates -√ Paused or failed deployment doesn't create any more canaries -√ Paused or failed deployment doesn't do any placements unless replacing lost allocs -√ Paused or failed deployment doesn't do destructive updates -√ Paused does do migrations -√ Failed deployment doesn't do migrations -√ Canary that is on a draining node -√ Canary that is on a lost node -√ Stop old canaries -√ Create new canaries on job change -√ Create new canaries on job change while scaling up -√ Create new canaries on job change while scaling down -√ Fill canaries if partial placement -√ Promote canaries unblocks max_parallel -√ Promote canaries when canaries == count -√ Only place as many as are healthy in deployment -√ Limit calculation accounts for healthy allocs on migrating/lost nodes -√ Failed deployment should not place anything -√ Run after canaries have been promoted, new allocs have been rolled out and there is no deployment -√ Failed deployment cancels non-promoted task groups -√ Failed deployment and updated job works -√ Finished deployment gets marked as complete -√ Change job change while scaling up -√ Update the job when all allocations from the previous job haven't been placed yet. -√ Paused or failed deployment doesn't do any rescheduling of failed allocs -√ Running deployment with failed allocs doesn't do any rescheduling of failed allocs -*/ - var ( canaryUpdate = &structs.UpdateStrategy{ Canary: 2, @@ -321,14 +258,14 @@ func assertResults(t *testing.T, r *reconcileResults, exp *resultExpectation) { assert := assert.New(t) if exp.createDeployment != nil && r.deployment == nil { - t.Fatalf("Expect a created deployment got none") + t.Errorf("Expect a created deployment got none") } else if exp.createDeployment == nil && r.deployment != nil { - t.Fatalf("Expect no created deployment; got %#v", r.deployment) + t.Errorf("Expect no created deployment; got %#v", r.deployment) } else if exp.createDeployment != nil && r.deployment != nil { // Clear the deployment ID r.deployment.ID, exp.createDeployment.ID = "", "" if !reflect.DeepEqual(r.deployment, exp.createDeployment) { - t.Fatalf("Unexpected createdDeployment; got\n %#v\nwant\n%#v\nDiff: %v", + t.Errorf("Unexpected createdDeployment; got\n %#v\nwant\n%#v\nDiff: %v", r.deployment, exp.createDeployment, pretty.Diff(r.deployment, exp.createDeployment)) } } @@ -1193,6 +1130,55 @@ func TestReconciler_MultiTG(t *testing.T) { assertNamesHaveIndexes(t, intRange(2, 9, 0, 9), placeResultsToNames(r.place)) } +// Tests the reconciler properly handles jobs with multiple task groups with +// only one having an update stanza and a deployment already being created +func TestReconciler_MultiTG_SingleUpdateStanza(t *testing.T) { + job := mock.Job() + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "foo" + job.TaskGroups = append(job.TaskGroups, tg2) + job.TaskGroups[0].Update = noCanaryUpdate + + // Create all the allocs + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + for j := 0; j < 10; j++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[i].Name, uint(j)) + alloc.TaskGroup = job.TaskGroups[i].Name + allocs = append(allocs, alloc) + } + } + + d := structs.NewDeployment(job) + d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ + DesiredTotal: 10, + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil, "") + r := reconciler.Compute() + + // Assert the correct results + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: nil, + place: 0, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Ignore: 10, + }, + tg2.Name: { + Ignore: 10, + }, + }, + }) +} + // Tests delayed rescheduling of failed batch allocations func TestReconciler_RescheduleLater_Batch(t *testing.T) { require := require.New(t) @@ -1881,6 +1867,362 @@ func TestReconciler_RescheduleNow_EvalIDMatch(t *testing.T) { assertPlacementsAreRescheduled(t, 1, r.place) } +// Tests rescheduling failed service allocations when there are canaries +func TestReconciler_RescheduleNow_Service_WithCanaries(t *testing.T) { + require := require.New(t) + + // Set desired 5 + job := mock.Job() + job.TaskGroups[0].Count = 5 + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Set up reschedule policy and update stanza + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 24 * time.Hour, + Delay: 5 * time.Second, + DelayFunction: "", + MaxDelay: 1 * time.Hour, + Unlimited: false, + } + job.TaskGroups[0].Update = canaryUpdate + + job2 := job.Copy() + job2.Version++ + + d := structs.NewDeployment(job2) + d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion + s := &structs.DeploymentState{ + DesiredCanaries: 2, + DesiredTotal: 5, + } + d.TaskGroups[job.TaskGroups[0].Name] = s + + // Create 5 existing allocations + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + allocs = append(allocs, alloc) + alloc.ClientStatus = structs.AllocClientStatusRunning + } + + // Mark three as failed + allocs[0].ClientStatus = structs.AllocClientStatusFailed + + // Mark one of them as already rescheduled once + allocs[0].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{ + {RescheduleTime: time.Now().Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + }, + }} + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "start", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed + + // Mark one as desired state stop + allocs[4].ClientStatus = structs.AllocClientStatusFailed + + // Create 2 canary allocations + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + Healthy: helper.BoolToPtr(false), + } + s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID) + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "") + r := reconciler.Compute() + + // Verify that no follow up evals were created + evals := r.desiredFollowupEvals[tgName] + require.Nil(evals) + + // Verify that one rescheduled alloc and one replacement for terminal alloc were placed + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: nil, + place: 2, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Place: 2, + Ignore: 5, + }, + }, + }) + + // Rescheduled allocs should have previous allocs + assertNamesHaveIndexes(t, intRange(1, 1, 4, 4), placeResultsToNames(r.place)) + assertPlaceResultsHavePreviousAllocs(t, 2, r.place) + assertPlacementsAreRescheduled(t, 2, r.place) +} + +// Tests rescheduling failed canary service allocations +func TestReconciler_RescheduleNow_Service_Canaries(t *testing.T) { + require := require.New(t) + + // Set desired 5 + job := mock.Job() + job.TaskGroups[0].Count = 5 + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Set up reschedule policy and update stanza + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Delay: 5 * time.Second, + DelayFunction: "constant", + MaxDelay: 1 * time.Hour, + Unlimited: true, + } + job.TaskGroups[0].Update = canaryUpdate + + job2 := job.Copy() + job2.Version++ + + d := structs.NewDeployment(job2) + d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion + s := &structs.DeploymentState{ + DesiredCanaries: 2, + DesiredTotal: 5, + } + d.TaskGroups[job.TaskGroups[0].Name] = s + + // Create 5 existing allocations + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + allocs = append(allocs, alloc) + alloc.ClientStatus = structs.AllocClientStatusRunning + } + + // Create 2 healthy canary allocations + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + Healthy: helper.BoolToPtr(false), + } + s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID) + allocs = append(allocs, alloc) + } + + // Mark the canaries as failed + allocs[5].ClientStatus = structs.AllocClientStatusFailed + allocs[5].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + // Mark one of them as already rescheduled once + allocs[5].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + }, + }} + + allocs[6].TaskStates = map[string]*structs.TaskState{tgName: {State: "start", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + allocs[6].ClientStatus = structs.AllocClientStatusFailed + allocs[6].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + // Create 4 unhealthy canary allocations that have already been replaced + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%2)) + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + Healthy: helper.BoolToPtr(false), + } + s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID) + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "") + reconciler.now = now + r := reconciler.Compute() + + // Verify that no follow up evals were created + evals := r.desiredFollowupEvals[tgName] + require.Nil(evals) + + // Verify that one rescheduled alloc and one replacement for terminal alloc were placed + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: nil, + place: 2, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Place: 2, + Ignore: 9, + }, + }, + }) + + // Rescheduled allocs should have previous allocs + assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) + assertPlaceResultsHavePreviousAllocs(t, 2, r.place) + assertPlacementsAreRescheduled(t, 2, r.place) +} + +// Tests rescheduling failed canary service allocations when one has reached its +// reschedule limit +func TestReconciler_RescheduleNow_Service_Canaries_Limit(t *testing.T) { + require := require.New(t) + + // Set desired 5 + job := mock.Job() + job.TaskGroups[0].Count = 5 + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Set up reschedule policy and update stanza + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 24 * time.Hour, + Delay: 5 * time.Second, + DelayFunction: "", + MaxDelay: 1 * time.Hour, + Unlimited: false, + } + job.TaskGroups[0].Update = canaryUpdate + + job2 := job.Copy() + job2.Version++ + + d := structs.NewDeployment(job2) + d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion + s := &structs.DeploymentState{ + DesiredCanaries: 2, + DesiredTotal: 5, + } + d.TaskGroups[job.TaskGroups[0].Name] = s + + // Create 5 existing allocations + var allocs []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + allocs = append(allocs, alloc) + alloc.ClientStatus = structs.AllocClientStatusRunning + } + + // Create 2 healthy canary allocations + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + Healthy: helper.BoolToPtr(false), + } + s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID) + allocs = append(allocs, alloc) + } + + // Mark the canaries as failed + allocs[5].ClientStatus = structs.AllocClientStatusFailed + allocs[5].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + // Mark one of them as already rescheduled once + allocs[5].RescheduleTracker = &structs.RescheduleTracker{Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + }, + }} + + allocs[6].TaskStates = map[string]*structs.TaskState{tgName: {State: "start", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + allocs[6].ClientStatus = structs.AllocClientStatusFailed + allocs[6].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + // Create 4 unhealthy canary allocations that have already been replaced + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%2)) + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{ + Canary: true, + Healthy: helper.BoolToPtr(false), + } + s.PlacedCanaries = append(s.PlacedCanaries, alloc.ID) + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job2, d, allocs, nil, "") + reconciler.now = now + r := reconciler.Compute() + + // Verify that no follow up evals were created + evals := r.desiredFollowupEvals[tgName] + require.Nil(evals) + + // Verify that one rescheduled alloc and one replacement for terminal alloc were placed + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: nil, + place: 1, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Place: 1, + Ignore: 10, + }, + }, + }) + + // Rescheduled allocs should have previous allocs + assertNamesHaveIndexes(t, intRange(1, 1), placeResultsToNames(r.place)) + assertPlaceResultsHavePreviousAllocs(t, 1, r.place) + assertPlacementsAreRescheduled(t, 1, r.place) +} + // Tests failed service allocations that were already rescheduled won't be rescheduled again func TestReconciler_DontReschedule_PreviouslyRescheduled(t *testing.T) { // Set desired 5 @@ -2810,6 +3152,55 @@ func TestReconciler_NewCanaries(t *testing.T) { assertNamesHaveIndexes(t, intRange(0, 1), placeResultsToNames(r.place)) } +// Tests the reconciler creates new canaries when the job changes and the +// canary count is greater than the task group count +func TestReconciler_NewCanaries_CountGreater(t *testing.T) { + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Update = canaryUpdate.Copy() + job.TaskGroups[0].Update.Canary = 7 + + // Create 3 allocations from the old job + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + alloc.TaskGroup = job.TaskGroups[0].Name + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, nil, allocs, nil, "") + r := reconciler.Compute() + + newD := structs.NewDeployment(job) + newD.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion + state := &structs.DeploymentState{ + DesiredCanaries: 7, + DesiredTotal: 3, + } + newD.TaskGroups[job.TaskGroups[0].Name] = state + + // Assert the correct results + assertResults(t, r, &resultExpectation{ + createDeployment: newD, + deploymentUpdates: nil, + place: 7, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Canary: 7, + Ignore: 3, + }, + }, + }) + + assertNamesHaveIndexes(t, intRange(0, 2, 3, 6), placeResultsToNames(r.place)) +} + // Tests the reconciler creates new canaries when the job changes for multiple // task groups func TestReconciler_NewCanaries_MultiTG(t *testing.T) { @@ -3117,6 +3508,7 @@ func TestReconciler_PromoteCanaries_CanariesEqualCount(t *testing.T) { DesiredTotal: 2, DesiredCanaries: 2, PlacedAllocs: 2, + HealthyAllocs: 2, } d.TaskGroups[job.TaskGroups[0].Name] = s @@ -3490,6 +3882,69 @@ func TestReconciler_CompleteDeployment(t *testing.T) { }) } +// Tests that the reconciler marks a deployment as complete once there is +// nothing left to place even if there are failed allocations that are part of +// the deployment. +func TestReconciler_MarkDeploymentComplete_FailedAllocations(t *testing.T) { + job := mock.Job() + job.TaskGroups[0].Update = noCanaryUpdate + + d := structs.NewDeployment(job) + d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ + DesiredTotal: 10, + PlacedAllocs: 20, + HealthyAllocs: 10, + } + + // Create 10 healthy allocs and 10 allocs that are failed + var allocs []*structs.Allocation + for i := 0; i < 20; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i%10)) + alloc.TaskGroup = job.TaskGroups[0].Name + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{} + if i < 10 { + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.DeploymentStatus.Healthy = helper.BoolToPtr(true) + } else { + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.DeploymentStatus.Healthy = helper.BoolToPtr(false) + } + + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnIgnore, false, job.ID, job, d, allocs, nil, "") + r := reconciler.Compute() + + updates := []*structs.DeploymentStatusUpdate{ + { + DeploymentID: d.ID, + Status: structs.DeploymentStatusSuccessful, + StatusDescription: structs.DeploymentStatusDescriptionSuccessful, + }, + } + + // Assert the correct results + assertResults(t, r, &resultExpectation{ + createDeployment: nil, + deploymentUpdates: updates, + place: 0, + inplace: 0, + stop: 0, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Ignore: 10, + }, + }, + }) +} + // Test that a failed deployment cancels non-promoted canaries func TestReconciler_FailedDeployment_CancelCanaries(t *testing.T) { // Create a job with two task groups @@ -3883,6 +4338,7 @@ func TestReconciler_FailedDeployment_DontReschedule(t *testing.T) { alloc.NodeID = uuid.Generate() alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) alloc.TaskGroup = job.TaskGroups[0].Name + alloc.DeploymentID = d.ID allocs = append(allocs, alloc) } @@ -3913,7 +4369,8 @@ func TestReconciler_FailedDeployment_DontReschedule(t *testing.T) { }) } -// Test that a running deployment with failed allocs will not result in rescheduling failed allocations +// Test that a running deployment with failed allocs will not result in +// rescheduling failed allocations unless they are marked as reschedulable. func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) { job := mock.Job() job.TaskGroups[0].Update = noCanaryUpdate @@ -3925,13 +4382,13 @@ func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) { d.Status = structs.DeploymentStatusRunning d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ Promoted: false, - DesiredTotal: 5, - PlacedAllocs: 4, + DesiredTotal: 10, + PlacedAllocs: 10, } - // Create 4 allocations and mark two as failed + // Create 10 allocations var allocs []*structs.Allocation - for i := 0; i < 4; i++ { + for i := 0; i < 10; i++ { alloc := mock.Alloc() alloc.Job = job alloc.JobID = job.ID @@ -3939,31 +4396,30 @@ func TestReconciler_DeploymentWithFailedAllocs_DontReschedule(t *testing.T) { alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) alloc.TaskGroup = job.TaskGroups[0].Name alloc.DeploymentID = d.ID + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "start", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} allocs = append(allocs, alloc) } - // Create allocs that are reschedulable now - allocs[2].ClientStatus = structs.AllocClientStatusFailed - allocs[2].TaskStates = map[string]*structs.TaskState{tgName: {State: "start", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - - allocs[3].ClientStatus = structs.AllocClientStatusFailed - allocs[3].TaskStates = map[string]*structs.TaskState{tgName: {State: "start", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} + // Mark half of them as reschedulable + for i := 0; i < 5; i++ { + allocs[i].DesiredTransition.Reschedule = helper.BoolToPtr(true) + } reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil, "") r := reconciler.Compute() // Assert that no rescheduled placements were created assertResults(t, r, &resultExpectation{ - place: 0, + place: 5, createDeployment: nil, deploymentUpdates: nil, desiredTGUpdates: map[string]*structs.DesiredUpdates{ job.TaskGroups[0].Name: { - Ignore: 2, + Place: 5, + Ignore: 5, }, }, }) @@ -3993,12 +4449,12 @@ func TestReconciler_FailedDeployment_AutoRevert_CancelCanaries(t *testing.T) { jobv2.Version = 2 jobv2.TaskGroups[0].Meta = map[string]string{"version": "2"} - // Create an existing failed deployment that has promoted one task group d := structs.NewDeployment(jobv2) state := &structs.DeploymentState{ - Promoted: false, - DesiredTotal: 3, - PlacedAllocs: 3, + Promoted: true, + DesiredTotal: 3, + PlacedAllocs: 3, + HealthyAllocs: 3, } d.TaskGroups[job.TaskGroups[0].Name] = state @@ -4062,3 +4518,55 @@ func TestReconciler_FailedDeployment_AutoRevert_CancelCanaries(t *testing.T) { }, }) } + +// Test that a successful deployment with failed allocs will result in +// rescheduling failed allocations +func TestReconciler_SuccessfulDeploymentWithFailedAllocs_Reschedule(t *testing.T) { + job := mock.Job() + job.TaskGroups[0].Update = noCanaryUpdate + tgName := job.TaskGroups[0].Name + now := time.Now() + + // Mock deployment with failed allocs, but deployment watcher hasn't marked it as failed yet + d := structs.NewDeployment(job) + d.Status = structs.DeploymentStatusSuccessful + d.TaskGroups[job.TaskGroups[0].Name] = &structs.DeploymentState{ + Promoted: false, + DesiredTotal: 10, + PlacedAllocs: 10, + } + + // Create 10 allocations + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = uuid.Generate() + alloc.Name = structs.AllocName(job.ID, job.TaskGroups[0].Name, uint(i)) + alloc.TaskGroup = job.TaskGroups[0].Name + alloc.DeploymentID = d.ID + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "start", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + allocs = append(allocs, alloc) + } + + reconciler := NewAllocReconciler(testLogger(), allocUpdateFnDestructive, false, job.ID, job, d, allocs, nil, "") + r := reconciler.Compute() + + // Assert that rescheduled placements were created + assertResults(t, r, &resultExpectation{ + place: 10, + createDeployment: nil, + deploymentUpdates: nil, + desiredTGUpdates: map[string]*structs.DesiredUpdates{ + job.TaskGroups[0].Name: { + Place: 10, + Ignore: 0, + }, + }, + }) + assertPlaceResultsHavePreviousAllocs(t, 10, r.place) +} diff --git a/scheduler/reconcile_util.go b/scheduler/reconcile_util.go index e64e8c9b5b45..b59fd8209cc2 100644 --- a/scheduler/reconcile_util.go +++ b/scheduler/reconcile_util.go @@ -234,7 +234,7 @@ func (a allocSet) filterByTainted(nodes map[string]*structs.Node) (untainted, mi // untainted or a set of allocations that must be rescheduled now. Allocations that can be rescheduled // at a future time are also returned so that we can create follow up evaluations for them. Allocs are // skipped or considered untainted according to logic defined in shouldFilter method. -func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID string) (untainted, rescheduleNow allocSet, rescheduleLater []*delayedRescheduleInfo) { +func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID string, deployment *structs.Deployment) (untainted, rescheduleNow allocSet, rescheduleLater []*delayedRescheduleInfo) { untainted = make(map[string]*structs.Allocation) rescheduleNow = make(map[string]*structs.Allocation) @@ -257,7 +257,7 @@ func (a allocSet) filterByRescheduleable(isBatch bool, now time.Time, evalID str // Only failed allocs with desired state run get to this point // If the failed alloc is not eligible for rescheduling now we add it to the untainted set - eligibleNow, eligibleLater, rescheduleTime = updateByReschedulable(alloc, now, evalID) + eligibleNow, eligibleLater, rescheduleTime = updateByReschedulable(alloc, now, evalID, deployment) if !eligibleNow { untainted[alloc.ID] = alloc if eligibleLater { @@ -320,9 +320,15 @@ func shouldFilter(alloc *structs.Allocation, isBatch bool) (untainted, ignore bo // updateByReschedulable is a helper method that encapsulates logic for whether a failed allocation // should be rescheduled now, later or left in the untainted set -func updateByReschedulable(alloc *structs.Allocation, now time.Time, evalID string) (rescheduleNow, rescheduleLater bool, rescheduleTime time.Time) { - rescheduleTime, eligible := alloc.NextRescheduleTime() +func updateByReschedulable(alloc *structs.Allocation, now time.Time, evalID string, d *structs.Deployment) (rescheduleNow, rescheduleLater bool, rescheduleTime time.Time) { + // If the allocation is part of an ongoing active deployment, we only allow it to reschedule + // if it has been marked eligible + if d != nil && alloc.DeploymentID == d.ID && d.Active() && !alloc.DesiredTransition.ShouldReschedule() { + return + } + // Reschedule if the eval ID matches the alloc's followup evalID or if its close to its reschedule time + rescheduleTime, eligible := alloc.NextRescheduleTime() if eligible && (alloc.FollowupEvalID == evalID || rescheduleTime.Sub(now) <= rescheduleWindowSize) { rescheduleNow = true return @@ -470,7 +476,7 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) [] // First select indexes from the allocations that are undergoing destructive // updates. This way we avoid duplicate names as they will get replaced. dmap := bitmapFrom(destructive, uint(a.count)) - var remainder uint + remainder := n for _, idx := range dmap.IndexesInRange(true, uint(0), uint(a.count)-1) { name := structs.AllocName(a.job, a.taskGroup, uint(idx)) if _, used := existingNames[name]; !used { @@ -478,7 +484,7 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) [] a.b.Set(uint(idx)) // If we have enough, return - remainder := n - uint(len(next)) + remainder = n - uint(len(next)) if remainder == 0 { return next } @@ -500,21 +506,15 @@ func (a *allocNameIndex) NextCanaries(n uint, existing, destructive allocSet) [] } } - // We have exhausted the preferred and free set, now just pick overlapping - // indexes - var i uint - for i = 0; i < remainder; i++ { + // We have exhausted the preferred and free set. Pick starting from n to + // n+remainder, to avoid overlapping where possible. An example is the + // desired count is 3 and we want 5 canaries. The first 3 canaries can use + // index [0, 1, 2] but after that we prefer picking indexes [4, 5] so that + // we do not overlap. Once the canaries are promoted, these would be the + // allocations that would be shut down as well. + for i := uint(a.count); i < uint(a.count)+remainder; i++ { name := structs.AllocName(a.job, a.taskGroup, i) - if _, used := existingNames[name]; !used { - next = append(next, name) - a.b.Set(i) - - // If we have enough, return - remainder = n - uint(len(next)) - if remainder == 0 { - return next - } - } + next = append(next, name) } return next diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go b/vendor/github.com/stretchr/testify/assert/assertion_format.go index 23838c4ceea7..677e19b20b56 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_format.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go @@ -13,6 +13,9 @@ import ( // Conditionf uses a Comparison to assert a complex condition. func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Condition(t, comp, append([]interface{}{msg}, args...)...) } @@ -22,19 +25,41 @@ func Conditionf(t TestingT, comp Comparison, msg string, args ...interface{}) bo // assert.Containsf(t, "Hello World", "World", "error message %s", "formatted") // assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") // assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Contains(t, s, contains, append([]interface{}{msg}, args...)...) } +// DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func DirExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return DirExists(t, path, append([]interface{}{msg}, args...)...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return ElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) +} + // Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // assert.Emptyf(t, obj, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Empty(t, object, append([]interface{}{msg}, args...)...) } @@ -42,12 +67,13 @@ func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) boo // // assert.Equalf(t, 123, 123, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Equal(t, expected, actual, append([]interface{}{msg}, args...)...) } @@ -56,9 +82,10 @@ func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, ar // // actualObj, err := SomeFunction() // assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return EqualError(t, theError, errString, append([]interface{}{msg}, args...)...) } @@ -66,9 +93,10 @@ func EqualErrorf(t TestingT, theError error, errString string, msg string, args // and equal. // // assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return EqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) } @@ -78,48 +106,68 @@ func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg stri // if assert.Errorf(t, err, "error message %s", "formatted") { // assert.Equal(t, expectedErrorf, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func Errorf(t TestingT, err error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Error(t, err, append([]interface{}{msg}, args...)...) } -// Exactlyf asserts that two objects are equal is value and type. +// Exactlyf asserts that two objects are equal in value and type. // // assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Exactly(t, expected, actual, append([]interface{}{msg}, args...)...) } // Failf reports a failure through func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, failureMessage, append([]interface{}{msg}, args...)...) } // FailNowf fails test func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return FailNow(t, failureMessage, append([]interface{}{msg}, args...)...) } // Falsef asserts that the specified value is false. // // assert.Falsef(t, myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Falsef(t TestingT, value bool, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return False(t, value, append([]interface{}{msg}, args...)...) } +// FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return FileExists(t, path, append([]interface{}{msg}, args...)...) +} + // HTTPBodyContainsf asserts that a specified handler returns a // body that contains a string. // // assert.HTTPBodyContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyContains(t, handler, method, url, values, str) +func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) } // HTTPBodyNotContainsf asserts that a specified handler returns a @@ -128,8 +176,11 @@ func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url // assert.HTTPBodyNotContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyNotContains(t, handler, method, url, values, str) +func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContains(t, handler, method, url, values, str, append([]interface{}{msg}, args...)...) } // HTTPErrorf asserts that a specified handler returns an error status code. @@ -137,8 +188,11 @@ func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, u // assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPError(t, handler, method, url, values) +func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPError(t, handler, method, url, values, append([]interface{}{msg}, args...)...) } // HTTPRedirectf asserts that a specified handler returns a redirect status code. @@ -146,8 +200,11 @@ func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, // assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPRedirect(t, handler, method, url, values) +func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPRedirect(t, handler, method, url, values, append([]interface{}{msg}, args...)...) } // HTTPSuccessf asserts that a specified handler returns a success status code. @@ -155,54 +212,80 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri // assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPSuccess(t, handler, method, url, values) +func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPSuccess(t, handler, method, url, values, append([]interface{}{msg}, args...)...) } // Implementsf asserts that an object is implemented by the specified interface. // // assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Implements(t, interfaceObject, object, append([]interface{}{msg}, args...)...) } // InDeltaf asserts that the two numerals are within delta of each other. // // assert.InDeltaf(t, math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return InDelta(t, expected, actual, delta, append([]interface{}{msg}, args...)...) } +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValues(t, expected, actual, delta, append([]interface{}{msg}, args...)...) +} + // InDeltaSlicef is the same as InDelta, except it compares two slices. func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return InDeltaSlice(t, expected, actual, delta, append([]interface{}{msg}, args...)...) } // InEpsilonf asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return InEpsilon(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) } // InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return InEpsilonSlice(t, expected, actual, epsilon, append([]interface{}{msg}, args...)...) } // IsTypef asserts that the specified objects are of the same type. func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return IsType(t, expectedType, object, append([]interface{}{msg}, args...)...) } // JSONEqf asserts that two JSON strings are equivalent. // // assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return JSONEq(t, expected, actual, append([]interface{}{msg}, args...)...) } @@ -210,18 +293,20 @@ func JSONEqf(t TestingT, expected string, actual string, msg string, args ...int // Lenf also fails if the object has a type that len() not accept. // // assert.Lenf(t, mySlice, 3, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Len(t, object, length, append([]interface{}{msg}, args...)...) } // Nilf asserts that the specified object is nil. // // assert.Nilf(t, err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Nil(t, object, append([]interface{}{msg}, args...)...) } @@ -231,9 +316,10 @@ func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) bool // if assert.NoErrorf(t, err, "error message %s", "formatted") { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NoError(t, err, append([]interface{}{msg}, args...)...) } @@ -243,9 +329,10 @@ func NoErrorf(t TestingT, err error, msg string, args ...interface{}) bool { // assert.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted") // assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") // assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotContains(t, s, contains, append([]interface{}{msg}, args...)...) } @@ -255,9 +342,10 @@ func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, a // if assert.NotEmptyf(t, obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotEmpty(t, object, append([]interface{}{msg}, args...)...) } @@ -265,29 +353,32 @@ func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) // // assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotEqual(t, expected, actual, append([]interface{}{msg}, args...)...) } // NotNilf asserts that the specified object is not nil. // // assert.NotNilf(t, err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotNil(t, object, append([]interface{}{msg}, args...)...) } // NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. // // assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotPanics(t, f, append([]interface{}{msg}, args...)...) } @@ -295,9 +386,10 @@ func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bo // // assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") // assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotRegexp(t, rx, str, append([]interface{}{msg}, args...)...) } @@ -305,23 +397,28 @@ func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args .. // elements given in the specified subset(array, slice...). // // assert.NotSubsetf(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotSubset(t, list, subset, append([]interface{}{msg}, args...)...) } -// NotZerof asserts that i is not the zero value for its type and returns the truth. +// NotZerof asserts that i is not the zero value for its type. func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return NotZero(t, i, append([]interface{}{msg}, args...)...) } // Panicsf asserts that the code inside the specified PanicTestFunc panics. // // assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Panics(t, f, append([]interface{}{msg}, args...)...) } @@ -329,9 +426,10 @@ func Panicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bool // the recovered panic value equals the expected panic value. // // assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return PanicsWithValue(t, expected, f, append([]interface{}{msg}, args...)...) } @@ -339,9 +437,10 @@ func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg str // // assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") // assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Regexp(t, rx, str, append([]interface{}{msg}, args...)...) } @@ -349,31 +448,37 @@ func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...in // elements given in the specified subset(array, slice...). // // assert.Subsetf(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Subset(t, list, subset, append([]interface{}{msg}, args...)...) } // Truef asserts that the specified value is true. // // assert.Truef(t, myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Truef(t TestingT, value bool, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return True(t, value, append([]interface{}{msg}, args...)...) } // WithinDurationf asserts that the two times are within duration delta of each other. // // assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return WithinDuration(t, expected, actual, delta, append([]interface{}{msg}, args...)...) } -// Zerof asserts that i is the zero value for its type and returns the truth. +// Zerof asserts that i is the zero value for its type. func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Zero(t, i, append([]interface{}{msg}, args...)...) } diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl b/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl index c5cc66f4305f..d2bb0b817788 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl +++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl @@ -1,4 +1,5 @@ {{.CommentFormat}} func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool { + if h, ok := t.(tHelper); ok { h.Helper() } return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}}) } diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/vendor/github.com/stretchr/testify/assert/assertion_forward.go index fcccbd01c8da..8adb111944c4 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go @@ -13,11 +13,17 @@ import ( // Condition uses a Comparison to assert a complex condition. func (a *Assertions) Condition(comp Comparison, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Condition(a.t, comp, msgAndArgs...) } // Conditionf uses a Comparison to assert a complex condition. func (a *Assertions) Conditionf(comp Comparison, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Conditionf(a.t, comp, msg, args...) } @@ -27,9 +33,10 @@ func (a *Assertions) Conditionf(comp Comparison, msg string, args ...interface{} // a.Contains("Hello World", "World") // a.Contains(["Hello", "World"], "World") // a.Contains({"Hello": "World"}, "Hello") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Contains(a.t, s, contains, msgAndArgs...) } @@ -39,19 +46,61 @@ func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs .. // a.Containsf("Hello World", "World", "error message %s", "formatted") // a.Containsf(["Hello", "World"], "World", "error message %s", "formatted") // a.Containsf({"Hello": "World"}, "Hello", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Containsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Containsf(a.t, s, contains, msg, args...) } +// DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return DirExists(a.t, path, msgAndArgs...) +} + +// DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return DirExistsf(a.t, path, msg, args...) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]) +func (a *Assertions) ElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatchf([1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return ElementsMatchf(a.t, listA, listB, msg, args...) +} + // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // a.Empty(obj) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Empty(a.t, object, msgAndArgs...) } @@ -59,9 +108,10 @@ func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { // a slice or a channel with len == 0. // // a.Emptyf(obj, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Emptyf(a.t, object, msg, args...) } @@ -69,12 +119,13 @@ func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) // // a.Equal(123, 123) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Equal(a.t, expected, actual, msgAndArgs...) } @@ -83,9 +134,10 @@ func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs // // actualObj, err := SomeFunction() // a.EqualError(err, expectedErrorString) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return EqualError(a.t, theError, errString, msgAndArgs...) } @@ -94,9 +146,10 @@ func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ... // // actualObj, err := SomeFunction() // a.EqualErrorf(err, expectedErrorString, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualErrorf(theError error, errString string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return EqualErrorf(a.t, theError, errString, msg, args...) } @@ -104,9 +157,10 @@ func (a *Assertions) EqualErrorf(theError error, errString string, msg string, a // and equal. // // a.EqualValues(uint32(123), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return EqualValues(a.t, expected, actual, msgAndArgs...) } @@ -114,9 +168,10 @@ func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAn // and equal. // // a.EqualValuesf(uint32(123, "error message %s", "formatted"), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return EqualValuesf(a.t, expected, actual, msg, args...) } @@ -124,12 +179,13 @@ func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg // // a.Equalf(123, 123, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Equalf(a.t, expected, actual, msg, args...) } @@ -139,9 +195,10 @@ func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string // if a.Error(err) { // assert.Equal(t, expectedError, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Error(a.t, err, msgAndArgs...) } @@ -151,76 +208,112 @@ func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { // if a.Errorf(err, "error message %s", "formatted") { // assert.Equal(t, expectedErrorf, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Errorf(err error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Errorf(a.t, err, msg, args...) } -// Exactly asserts that two objects are equal is value and type. +// Exactly asserts that two objects are equal in value and type. // // a.Exactly(int32(123), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Exactly(a.t, expected, actual, msgAndArgs...) } -// Exactlyf asserts that two objects are equal is value and type. +// Exactlyf asserts that two objects are equal in value and type. // // a.Exactlyf(int32(123, "error message %s", "formatted"), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Exactlyf(a.t, expected, actual, msg, args...) } // Fail reports a failure through func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Fail(a.t, failureMessage, msgAndArgs...) } // FailNow fails test func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return FailNow(a.t, failureMessage, msgAndArgs...) } // FailNowf fails test func (a *Assertions) FailNowf(failureMessage string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return FailNowf(a.t, failureMessage, msg, args...) } // Failf reports a failure through func (a *Assertions) Failf(failureMessage string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Failf(a.t, failureMessage, msg, args...) } // False asserts that the specified value is false. // // a.False(myBool) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) False(value bool, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return False(a.t, value, msgAndArgs...) } // Falsef asserts that the specified value is false. // // a.Falsef(myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Falsef(value bool, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Falsef(a.t, value, msg, args...) } +// FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExists(path string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FileExists(a.t, path, msgAndArgs...) +} + +// FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return FileExistsf(a.t, path, msg, args...) +} + // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // // a.HTTPBodyContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyContains(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContains(a.t, handler, method, url, values, str, msgAndArgs...) } // HTTPBodyContainsf asserts that a specified handler returns a @@ -229,8 +322,11 @@ func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, u // a.HTTPBodyContainsf(myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyContainsf(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyContainsf(a.t, handler, method, url, values, str, msg, args...) } // HTTPBodyNotContains asserts that a specified handler returns a @@ -239,8 +335,11 @@ func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, // a.HTTPBodyNotContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyNotContains(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContains(a.t, handler, method, url, values, str, msgAndArgs...) } // HTTPBodyNotContainsf asserts that a specified handler returns a @@ -249,8 +348,11 @@ func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string // a.HTTPBodyNotContainsf(myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) bool { - return HTTPBodyNotContainsf(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPBodyNotContainsf(a.t, handler, method, url, values, str, msg, args...) } // HTTPError asserts that a specified handler returns an error status code. @@ -258,8 +360,11 @@ func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method strin // a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPError(a.t, handler, method, url, values) +func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPError(a.t, handler, method, url, values, msgAndArgs...) } // HTTPErrorf asserts that a specified handler returns an error status code. @@ -267,8 +372,11 @@ func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url stri // a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPErrorf(a.t, handler, method, url, values) +func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPErrorf(a.t, handler, method, url, values, msg, args...) } // HTTPRedirect asserts that a specified handler returns a redirect status code. @@ -276,8 +384,11 @@ func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url str // a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPRedirect(a.t, handler, method, url, values) +func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPRedirect(a.t, handler, method, url, values, msgAndArgs...) } // HTTPRedirectf asserts that a specified handler returns a redirect status code. @@ -285,8 +396,11 @@ func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url s // a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPRedirectf(a.t, handler, method, url, values) +func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPRedirectf(a.t, handler, method, url, values, msg, args...) } // HTTPSuccess asserts that a specified handler returns a success status code. @@ -294,8 +408,11 @@ func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url // a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPSuccess(a.t, handler, method, url, values) +func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPSuccess(a.t, handler, method, url, values, msgAndArgs...) } // HTTPSuccessf asserts that a specified handler returns a success status code. @@ -303,14 +420,20 @@ func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url st // a.HTTPSuccessf(myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values) bool { - return HTTPSuccessf(a.t, handler, method, url, values) +func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPSuccessf(a.t, handler, method, url, values, msg, args...) } // Implements asserts that an object is implemented by the specified interface. // // a.Implements((*MyInterface)(nil), new(MyObject)) func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Implements(a.t, interfaceObject, object, msgAndArgs...) } @@ -318,86 +441,129 @@ func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, // // a.Implementsf((*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Implementsf(a.t, interfaceObject, object, msg, args...) } // InDelta asserts that the two numerals are within delta of each other. // // a.InDelta(math.Pi, (22 / 7.0), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InDelta(a.t, expected, actual, delta, msgAndArgs...) } +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValues(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValues(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValuesf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return InDeltaMapValuesf(a.t, expected, actual, delta, msg, args...) +} + // InDeltaSlice is the same as InDelta, except it compares two slices. func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) } // InDeltaSlicef is the same as InDelta, except it compares two slices. func (a *Assertions) InDeltaSlicef(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InDeltaSlicef(a.t, expected, actual, delta, msg, args...) } // InDeltaf asserts that the two numerals are within delta of each other. // // a.InDeltaf(math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InDeltaf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InDeltaf(a.t, expected, actual, delta, msg, args...) } // InEpsilon asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) } // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InEpsilonSlice(a.t, expected, actual, epsilon, msgAndArgs...) } // InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. func (a *Assertions) InEpsilonSlicef(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InEpsilonSlicef(a.t, expected, actual, epsilon, msg, args...) } // InEpsilonf asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return InEpsilonf(a.t, expected, actual, epsilon, msg, args...) } // IsType asserts that the specified objects are of the same type. func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return IsType(a.t, expectedType, object, msgAndArgs...) } // IsTypef asserts that the specified objects are of the same type. func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return IsTypef(a.t, expectedType, object, msg, args...) } // JSONEq asserts that two JSON strings are equivalent. // // a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return JSONEq(a.t, expected, actual, msgAndArgs...) } // JSONEqf asserts that two JSON strings are equivalent. // // a.JSONEqf(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) JSONEqf(expected string, actual string, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return JSONEqf(a.t, expected, actual, msg, args...) } @@ -405,9 +571,10 @@ func (a *Assertions) JSONEqf(expected string, actual string, msg string, args .. // Len also fails if the object has a type that len() not accept. // // a.Len(mySlice, 3) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Len(a.t, object, length, msgAndArgs...) } @@ -415,27 +582,30 @@ func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface // Lenf also fails if the object has a type that len() not accept. // // a.Lenf(mySlice, 3, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Lenf(a.t, object, length, msg, args...) } // Nil asserts that the specified object is nil. // // a.Nil(err) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Nil(a.t, object, msgAndArgs...) } // Nilf asserts that the specified object is nil. // // a.Nilf(err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Nilf(a.t, object, msg, args...) } @@ -445,9 +615,10 @@ func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) b // if a.NoError(err) { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NoError(a.t, err, msgAndArgs...) } @@ -457,9 +628,10 @@ func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) bool { // if a.NoErrorf(err, "error message %s", "formatted") { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NoErrorf(a.t, err, msg, args...) } @@ -469,9 +641,10 @@ func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) bool { // a.NotContains("Hello World", "Earth") // a.NotContains(["Hello", "World"], "Earth") // a.NotContains({"Hello": "World"}, "Earth") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotContains(a.t, s, contains, msgAndArgs...) } @@ -481,9 +654,10 @@ func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs // a.NotContainsf("Hello World", "Earth", "error message %s", "formatted") // a.NotContainsf(["Hello", "World"], "Earth", "error message %s", "formatted") // a.NotContainsf({"Hello": "World"}, "Earth", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotContainsf(a.t, s, contains, msg, args...) } @@ -493,9 +667,10 @@ func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg strin // if a.NotEmpty(obj) { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotEmpty(a.t, object, msgAndArgs...) } @@ -505,9 +680,10 @@ func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) boo // if a.NotEmptyf(obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotEmptyf(a.t, object, msg, args...) } @@ -515,11 +691,12 @@ func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface // // a.NotEqual(obj1, obj2) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotEqual(a.t, expected, actual, msgAndArgs...) } @@ -527,47 +704,52 @@ func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndAr // // a.NotEqualf(obj1, obj2, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotEqualf(a.t, expected, actual, msg, args...) } // NotNil asserts that the specified object is not nil. // // a.NotNil(err) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotNil(a.t, object, msgAndArgs...) } // NotNilf asserts that the specified object is not nil. // // a.NotNilf(err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotNilf(object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotNilf(a.t, object, msg, args...) } // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // a.NotPanics(func(){ RemainCalm() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotPanics(f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotPanics(a.t, f, msgAndArgs...) } // NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. // // a.NotPanicsf(func(){ RemainCalm() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotPanicsf(f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotPanicsf(a.t, f, msg, args...) } @@ -575,9 +757,10 @@ func (a *Assertions) NotPanicsf(f PanicTestFunc, msg string, args ...interface{} // // a.NotRegexp(regexp.MustCompile("starts"), "it's starting") // a.NotRegexp("^start", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotRegexp(a.t, rx, str, msgAndArgs...) } @@ -585,9 +768,10 @@ func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...in // // a.NotRegexpf(regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") // a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotRegexpf(a.t, rx, str, msg, args...) } @@ -595,9 +779,10 @@ func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, arg // elements given in the specified subset(array, slice...). // // a.NotSubset([1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotSubset(a.t, list, subset, msgAndArgs...) } @@ -605,28 +790,36 @@ func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs // elements given in the specified subset(array, slice...). // // a.NotSubsetf([1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotSubsetf(a.t, list, subset, msg, args...) } -// NotZero asserts that i is not the zero value for its type and returns the truth. +// NotZero asserts that i is not the zero value for its type. func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotZero(a.t, i, msgAndArgs...) } -// NotZerof asserts that i is not the zero value for its type and returns the truth. +// NotZerof asserts that i is not the zero value for its type. func (a *Assertions) NotZerof(i interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return NotZerof(a.t, i, msg, args...) } // Panics asserts that the code inside the specified PanicTestFunc panics. // // a.Panics(func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Panics(f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Panics(a.t, f, msgAndArgs...) } @@ -634,9 +827,10 @@ func (a *Assertions) Panics(f PanicTestFunc, msgAndArgs ...interface{}) bool { // the recovered panic value equals the expected panic value. // // a.PanicsWithValue("crazy error", func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) PanicsWithValue(expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return PanicsWithValue(a.t, expected, f, msgAndArgs...) } @@ -644,18 +838,20 @@ func (a *Assertions) PanicsWithValue(expected interface{}, f PanicTestFunc, msgA // the recovered panic value equals the expected panic value. // // a.PanicsWithValuef("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) PanicsWithValuef(expected interface{}, f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return PanicsWithValuef(a.t, expected, f, msg, args...) } // Panicsf asserts that the code inside the specified PanicTestFunc panics. // // a.Panicsf(func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Panicsf(f PanicTestFunc, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Panicsf(a.t, f, msg, args...) } @@ -663,9 +859,10 @@ func (a *Assertions) Panicsf(f PanicTestFunc, msg string, args ...interface{}) b // // a.Regexp(regexp.MustCompile("start"), "it's starting") // a.Regexp("start...$", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Regexp(a.t, rx, str, msgAndArgs...) } @@ -673,9 +870,10 @@ func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...inter // // a.Regexpf(regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") // a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Regexpf(a.t, rx, str, msg, args...) } @@ -683,9 +881,10 @@ func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args . // elements given in the specified subset(array, slice...). // // a.Subset([1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Subset(a.t, list, subset, msgAndArgs...) } @@ -693,54 +892,65 @@ func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ... // elements given in the specified subset(array, slice...). // // a.Subsetf([1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Subsetf(a.t, list, subset, msg, args...) } // True asserts that the specified value is true. // // a.True(myBool) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) True(value bool, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return True(a.t, value, msgAndArgs...) } // Truef asserts that the specified value is true. // // a.Truef(myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Truef(value bool, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Truef(a.t, value, msg, args...) } // WithinDuration asserts that the two times are within duration delta of each other. // // a.WithinDuration(time.Now(), time.Now(), 10*time.Second) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return WithinDuration(a.t, expected, actual, delta, msgAndArgs...) } // WithinDurationf asserts that the two times are within duration delta of each other. // // a.WithinDurationf(time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return WithinDurationf(a.t, expected, actual, delta, msg, args...) } -// Zero asserts that i is the zero value for its type and returns the truth. +// Zero asserts that i is the zero value for its type. func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Zero(a.t, i, msgAndArgs...) } -// Zerof asserts that i is the zero value for its type and returns the truth. +// Zerof asserts that i is the zero value for its type. func (a *Assertions) Zerof(i interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } return Zerof(a.t, i, msg, args...) } diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl b/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl index 99f9acfbba5f..188bb9e17439 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl +++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl @@ -1,4 +1,5 @@ {{.CommentWithoutT "a"}} func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { + if h, ok := a.t.(tHelper); ok { h.Helper() } return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) } diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go index c8034f68ae49..94781ea339d5 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/vendor/github.com/stretchr/testify/assert/assertions.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "os" "reflect" "regexp" "runtime" @@ -26,6 +27,22 @@ type TestingT interface { Errorf(format string, args ...interface{}) } +// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful +// for table driven tests. +type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) bool + +// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful +// for table driven tests. +type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool + +// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful +// for table driven tests. +type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool + +// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// for table driven tests. +type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool + // Comparison a custom function that returns true on success and false on failure type Comparison func() (success bool) @@ -155,21 +172,6 @@ func isTest(name, prefix string) bool { return !unicode.IsLower(rune) } -// getWhitespaceString returns a string that is long enough to overwrite the default -// output from the go testing framework. -func getWhitespaceString() string { - - _, file, line, ok := runtime.Caller(1) - if !ok { - return "" - } - parts := strings.Split(file, "/") - file = parts[len(parts)-1] - - return strings.Repeat(" ", len(fmt.Sprintf("%s:%d: ", file, line))) - -} - func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { if len(msgAndArgs) == 0 || msgAndArgs == nil { return "" @@ -194,7 +196,7 @@ func indentMessageLines(message string, longestLabelLen int) string { // no need to align first line because it starts at the correct location (after the label) if i != 0 { // append alignLen+1 spaces to align with "{{longestLabel}}:" before adding tab - outBuf.WriteString("\n\r\t" + strings.Repeat(" ", longestLabelLen+1) + "\t") + outBuf.WriteString("\n\t" + strings.Repeat(" ", longestLabelLen+1) + "\t") } outBuf.WriteString(scanner.Text()) } @@ -208,6 +210,9 @@ type failNower interface { // FailNow fails test func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } Fail(t, failureMessage, msgAndArgs...) // We cannot extend TestingT with FailNow() and @@ -226,17 +231,27 @@ func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool // Fail reports a failure through func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } content := []labeledContent{ - {"Error Trace", strings.Join(CallerInfo(), "\n\r\t\t\t")}, + {"Error Trace", strings.Join(CallerInfo(), "\n\t\t\t")}, {"Error", failureMessage}, } + // Add test name if the Go version supports it + if n, ok := t.(interface { + Name() string + }); ok { + content = append(content, labeledContent{"Test", n.Name()}) + } + message := messageFromMsgAndArgs(msgAndArgs...) if len(message) > 0 { content = append(content, labeledContent{"Messages", message}) } - t.Errorf("%s", "\r"+getWhitespaceString()+labeledOutput(content...)) + t.Errorf("\n%s", ""+labeledOutput(content...)) return false } @@ -248,7 +263,7 @@ type labeledContent struct { // labeledOutput returns a string consisting of the provided labeledContent. Each labeled output is appended in the following manner: // -// \r\t{{label}}:{{align_spaces}}\t{{content}}\n +// \t{{label}}:{{align_spaces}}\t{{content}}\n // // The initial carriage return is required to undo/erase any padding added by testing.T.Errorf. The "\t{{label}}:" is for the label. // If a label is shorter than the longest label provided, padding spaces are added to make all the labels match in length. Once this @@ -264,7 +279,7 @@ func labeledOutput(content ...labeledContent) string { } var output string for _, v := range content { - output += "\r\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n" + output += "\t" + v.label + ":" + strings.Repeat(" ", longestLabel-len(v.label)) + "\t" + indentMessageLines(v.content, longestLabel) + "\n" } return output } @@ -273,19 +288,26 @@ func labeledOutput(content ...labeledContent) string { // // assert.Implements(t, (*MyInterface)(nil), new(MyObject)) func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) bool { - + if h, ok := t.(tHelper); ok { + h.Helper() + } interfaceType := reflect.TypeOf(interfaceObject).Elem() + if object == nil { + return Fail(t, fmt.Sprintf("Cannot check if nil implements %v", interfaceType), msgAndArgs...) + } if !reflect.TypeOf(object).Implements(interfaceType) { return Fail(t, fmt.Sprintf("%T must implement %v", object, interfaceType), msgAndArgs...) } return true - } // IsType asserts that the specified objects are of the same type. func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) { return Fail(t, fmt.Sprintf("Object expected to be of type %v, but was %v", reflect.TypeOf(expectedType), reflect.TypeOf(object)), msgAndArgs...) @@ -298,12 +320,13 @@ func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs // // assert.Equal(t, 123, 123) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if err := validateEqualArgs(expected, actual); err != nil { return Fail(t, fmt.Sprintf("Invalid operation: %#v == %#v (%s)", expected, actual, err), msgAndArgs...) @@ -314,7 +337,7 @@ func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) expected, actual = formatUnequalValues(expected, actual) return Fail(t, fmt.Sprintf("Not equal: \n"+ "expected: %s\n"+ - "actual: %s%s", expected, actual, diff), msgAndArgs...) + "actual : %s%s", expected, actual, diff), msgAndArgs...) } return true @@ -341,34 +364,36 @@ func formatUnequalValues(expected, actual interface{}) (e string, a string) { // and equal. // // assert.EqualValues(t, uint32(123), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !ObjectsAreEqualValues(expected, actual) { diff := diff(expected, actual) expected, actual = formatUnequalValues(expected, actual) return Fail(t, fmt.Sprintf("Not equal: \n"+ "expected: %s\n"+ - "actual: %s%s", expected, actual, diff), msgAndArgs...) + "actual : %s%s", expected, actual, diff), msgAndArgs...) } return true } -// Exactly asserts that two objects are equal is value and type. +// Exactly asserts that two objects are equal in value and type. // // assert.Exactly(t, int32(123), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } aType := reflect.TypeOf(expected) bType := reflect.TypeOf(actual) if aType != bType { - return Fail(t, fmt.Sprintf("Types expected to match exactly\n\r\t%v != %v", aType, bType), msgAndArgs...) + return Fail(t, fmt.Sprintf("Types expected to match exactly\n\t%v != %v", aType, bType), msgAndArgs...) } return Equal(t, expected, actual, msgAndArgs...) @@ -378,9 +403,10 @@ func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{} // NotNil asserts that the specified object is not nil. // // assert.NotNil(t, err) -// -// Returns whether the assertion was successful (true) or not (false). func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !isNil(object) { return true } @@ -405,84 +431,52 @@ func isNil(object interface{}) bool { // Nil asserts that the specified object is nil. // // assert.Nil(t, err) -// -// Returns whether the assertion was successful (true) or not (false). func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if isNil(object) { return true } return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...) } -var numericZeros = []interface{}{ - int(0), - int8(0), - int16(0), - int32(0), - int64(0), - uint(0), - uint8(0), - uint16(0), - uint32(0), - uint64(0), - float32(0), - float64(0), -} - // isEmpty gets whether the specified object is considered empty or not. func isEmpty(object interface{}) bool { + // get nil case out of the way if object == nil { return true - } else if object == "" { - return true - } else if object == false { - return true - } - - for _, v := range numericZeros { - if object == v { - return true - } } objValue := reflect.ValueOf(object) switch objValue.Kind() { - case reflect.Map: - fallthrough - case reflect.Slice, reflect.Chan: - { - return (objValue.Len() == 0) - } - case reflect.Struct: - switch object.(type) { - case time.Time: - return object.(time.Time).IsZero() - } + // collection types are empty when they have no element + case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice: + return objValue.Len() == 0 + // pointers are empty if nil or if the value they point to is empty case reflect.Ptr: - { - if objValue.IsNil() { - return true - } - switch object.(type) { - case *time.Time: - return object.(*time.Time).IsZero() - default: - return false - } + if objValue.IsNil() { + return true } + deref := objValue.Elem().Interface() + return isEmpty(deref) + // for all other types, compare against the zero value + default: + zero := reflect.Zero(objValue.Type()) + return reflect.DeepEqual(object, zero.Interface()) } - return false } // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // assert.Empty(t, obj) -// -// Returns whether the assertion was successful (true) or not (false). func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } pass := isEmpty(object) if !pass { @@ -499,9 +493,10 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { // if assert.NotEmpty(t, obj) { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } pass := !isEmpty(object) if !pass { @@ -528,9 +523,10 @@ func getLen(x interface{}) (ok bool, length int) { // Len also fails if the object has a type that len() not accept. // // assert.Len(t, mySlice, 3) -// -// Returns whether the assertion was successful (true) or not (false). func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } ok, l := getLen(object) if !ok { return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", object), msgAndArgs...) @@ -545,9 +541,15 @@ func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) // True asserts that the specified value is true. // // assert.True(t, myBool) -// -// Returns whether the assertion was successful (true) or not (false). func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if h, ok := t.(interface { + Helper() + }); ok { + h.Helper() + } if value != true { return Fail(t, "Should be true", msgAndArgs...) @@ -560,9 +562,10 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { // False asserts that the specified value is false. // // assert.False(t, myBool) -// -// Returns whether the assertion was successful (true) or not (false). func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if value != false { return Fail(t, "Should be false", msgAndArgs...) @@ -576,11 +579,12 @@ func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { // // assert.NotEqual(t, obj1, obj2) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if err := validateEqualArgs(expected, actual); err != nil { return Fail(t, fmt.Sprintf("Invalid operation: %#v != %#v (%s)", expected, actual, err), msgAndArgs...) @@ -638,9 +642,10 @@ func includeElement(list interface{}, element interface{}) (ok, found bool) { // assert.Contains(t, "Hello World", "World") // assert.Contains(t, ["Hello", "World"], "World") // assert.Contains(t, {"Hello": "World"}, "Hello") -// -// Returns whether the assertion was successful (true) or not (false). func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } ok, found := includeElement(s, contains) if !ok { @@ -660,9 +665,10 @@ func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bo // assert.NotContains(t, "Hello World", "Earth") // assert.NotContains(t, ["Hello", "World"], "Earth") // assert.NotContains(t, {"Hello": "World"}, "Earth") -// -// Returns whether the assertion was successful (true) or not (false). func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } ok, found := includeElement(s, contains) if !ok { @@ -680,9 +686,10 @@ func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) // elements given in the specified subset(array, slice...). // // assert.Subset(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if subset == nil { return true // we consider nil to be equal to the nil set } @@ -723,11 +730,12 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok // elements given in the specified subset(array, slice...). // // assert.NotSubset(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if subset == nil { - return false // we consider nil to be equal to the nil set + return Fail(t, fmt.Sprintf("nil is the empty set which is a subset of every set"), msgAndArgs...) } subsetValue := reflect.ValueOf(subset) @@ -762,8 +770,68 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) return Fail(t, fmt.Sprintf("%q is a subset of %q", subset, list), msgAndArgs...) } +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) +func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface{}) (ok bool) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if isEmpty(listA) && isEmpty(listB) { + return true + } + + aKind := reflect.TypeOf(listA).Kind() + bKind := reflect.TypeOf(listB).Kind() + + if aKind != reflect.Array && aKind != reflect.Slice { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listA, aKind), msgAndArgs...) + } + + if bKind != reflect.Array && bKind != reflect.Slice { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listB, bKind), msgAndArgs...) + } + + aValue := reflect.ValueOf(listA) + bValue := reflect.ValueOf(listB) + + aLen := aValue.Len() + bLen := bValue.Len() + + if aLen != bLen { + return Fail(t, fmt.Sprintf("lengths don't match: %d != %d", aLen, bLen), msgAndArgs...) + } + + // Mark indexes in bValue that we already used + visited := make([]bool, bLen) + for i := 0; i < aLen; i++ { + element := aValue.Index(i).Interface() + found := false + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + if ObjectsAreEqual(bValue.Index(j).Interface(), element) { + visited[j] = true + found = true + break + } + } + if !found { + return Fail(t, fmt.Sprintf("element %s appears more times in %s than in %s", element, aValue, bValue), msgAndArgs...) + } + } + + return true +} + // Condition uses a Comparison to assert a complex condition. func Condition(t TestingT, comp Comparison, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } result := comp() if !result { Fail(t, "Condition failed!", msgAndArgs...) @@ -800,12 +868,13 @@ func didPanic(f PanicTestFunc) (bool, interface{}) { // Panics asserts that the code inside the specified PanicTestFunc panics. // // assert.Panics(t, func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if funcDidPanic, panicValue := didPanic(f); !funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...) + return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...) } return true @@ -815,16 +884,17 @@ func Panics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { // the recovered panic value equals the expected panic value. // // assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } funcDidPanic, panicValue := didPanic(f) if !funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...) + return Fail(t, fmt.Sprintf("func %#v should panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...) } if panicValue != expected { - return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%v\n\r\tPanic value:\t%v", f, expected, panicValue), msgAndArgs...) + return Fail(t, fmt.Sprintf("func %#v should panic with value:\t%v\n\tPanic value:\t%v", f, expected, panicValue), msgAndArgs...) } return true @@ -833,12 +903,13 @@ func PanicsWithValue(t TestingT, expected interface{}, f PanicTestFunc, msgAndAr // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // assert.NotPanics(t, func(){ RemainCalm() }) -// -// Returns whether the assertion was successful (true) or not (false). func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if funcDidPanic, panicValue := didPanic(f); funcDidPanic { - return Fail(t, fmt.Sprintf("func %#v should not panic\n\r\tPanic value:\t%v", f, panicValue), msgAndArgs...) + return Fail(t, fmt.Sprintf("func %#v should not panic\n\tPanic value:\t%v", f, panicValue), msgAndArgs...) } return true @@ -847,9 +918,10 @@ func NotPanics(t TestingT, f PanicTestFunc, msgAndArgs ...interface{}) bool { // WithinDuration asserts that the two times are within duration delta of each other. // // assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) -// -// Returns whether the assertion was successful (true) or not (false). func WithinDuration(t TestingT, expected, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } dt := expected.Sub(actual) if dt < -delta || dt > delta { @@ -886,6 +958,8 @@ func toFloat(x interface{}) (float64, bool) { xf = float64(xn) case float64: xf = float64(xn) + case time.Duration: + xf = float64(xn) default: xok = false } @@ -896,9 +970,10 @@ func toFloat(x interface{}) (float64, bool) { // InDelta asserts that the two numerals are within delta of each other. // // assert.InDelta(t, math.Pi, (22 / 7.0), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } af, aok := toFloat(expected) bf, bok := toFloat(actual) @@ -908,7 +983,7 @@ func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs } if math.IsNaN(af) { - return Fail(t, fmt.Sprintf("Actual must not be NaN"), msgAndArgs...) + return Fail(t, fmt.Sprintf("Expected must not be NaN"), msgAndArgs...) } if math.IsNaN(bf) { @@ -925,6 +1000,9 @@ func InDelta(t TestingT, expected, actual interface{}, delta float64, msgAndArgs // InDeltaSlice is the same as InDelta, except it compares two slices. func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if expected == nil || actual == nil || reflect.TypeOf(actual).Kind() != reflect.Slice || reflect.TypeOf(expected).Kind() != reflect.Slice { @@ -944,6 +1022,50 @@ func InDeltaSlice(t TestingT, expected, actual interface{}, delta float64, msgAn return true } +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValues(t TestingT, expected, actual interface{}, delta float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if expected == nil || actual == nil || + reflect.TypeOf(actual).Kind() != reflect.Map || + reflect.TypeOf(expected).Kind() != reflect.Map { + return Fail(t, "Arguments must be maps", msgAndArgs...) + } + + expectedMap := reflect.ValueOf(expected) + actualMap := reflect.ValueOf(actual) + + if expectedMap.Len() != actualMap.Len() { + return Fail(t, "Arguments must have the same number of keys", msgAndArgs...) + } + + for _, k := range expectedMap.MapKeys() { + ev := expectedMap.MapIndex(k) + av := actualMap.MapIndex(k) + + if !ev.IsValid() { + return Fail(t, fmt.Sprintf("missing key %q in expected map", k), msgAndArgs...) + } + + if !av.IsValid() { + return Fail(t, fmt.Sprintf("missing key %q in actual map", k), msgAndArgs...) + } + + if !InDelta( + t, + ev.Interface(), + av.Interface(), + delta, + msgAndArgs..., + ) { + return false + } + } + + return true +} + func calcRelativeError(expected, actual interface{}) (float64, error) { af, aok := toFloat(expected) if !aok { @@ -954,23 +1076,24 @@ func calcRelativeError(expected, actual interface{}) (float64, error) { } bf, bok := toFloat(actual) if !bok { - return 0, fmt.Errorf("expected value %q cannot be converted to float", actual) + return 0, fmt.Errorf("actual value %q cannot be converted to float", actual) } return math.Abs(af-bf) / math.Abs(af), nil } // InEpsilon asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } actualEpsilon, err := calcRelativeError(expected, actual) if err != nil { return Fail(t, err.Error(), msgAndArgs...) } if actualEpsilon > epsilon { return Fail(t, fmt.Sprintf("Relative error is too high: %#v (expected)\n"+ - " < %#v (actual)", actualEpsilon, epsilon), msgAndArgs...) + " < %#v (actual)", epsilon, actualEpsilon), msgAndArgs...) } return true @@ -978,6 +1101,9 @@ func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAnd // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if expected == nil || actual == nil || reflect.TypeOf(actual).Kind() != reflect.Slice || reflect.TypeOf(expected).Kind() != reflect.Slice { @@ -1007,9 +1133,10 @@ func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, m // if assert.NoError(t, err) { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if err != nil { return Fail(t, fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...) } @@ -1023,9 +1150,10 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { // if assert.Error(t, err) { // assert.Equal(t, expectedError, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if err == nil { return Fail(t, "An error is expected but got nil.", msgAndArgs...) @@ -1039,9 +1167,10 @@ func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { // // actualObj, err := SomeFunction() // assert.EqualError(t, err, expectedErrorString) -// -// Returns whether the assertion was successful (true) or not (false). func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !Error(t, theError, msgAndArgs...) { return false } @@ -1051,7 +1180,7 @@ func EqualError(t TestingT, theError error, errString string, msgAndArgs ...inte if expected != actual { return Fail(t, fmt.Sprintf("Error message not equal:\n"+ "expected: %q\n"+ - "actual: %q", expected, actual), msgAndArgs...) + "actual : %q", expected, actual), msgAndArgs...) } return true } @@ -1074,9 +1203,10 @@ func matchRegexp(rx interface{}, str interface{}) bool { // // assert.Regexp(t, regexp.MustCompile("start"), "it's starting") // assert.Regexp(t, "start...$", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } match := matchRegexp(rx, str) @@ -1091,9 +1221,10 @@ func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface // // assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") // assert.NotRegexp(t, "^start", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } match := matchRegexp(rx, str) if match { @@ -1104,28 +1235,71 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf } -// Zero asserts that i is the zero value for its type and returns the truth. +// Zero asserts that i is the zero value for its type. func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if i != nil && !reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { return Fail(t, fmt.Sprintf("Should be zero, but was %v", i), msgAndArgs...) } return true } -// NotZero asserts that i is not the zero value for its type and returns the truth. +// NotZero asserts that i is not the zero value for its type. func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } if i == nil || reflect.DeepEqual(i, reflect.Zero(reflect.TypeOf(i)).Interface()) { return Fail(t, fmt.Sprintf("Should not be zero, but was %v", i), msgAndArgs...) } return true } +// FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func FileExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) + } + return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) + } + if info.IsDir() { + return Fail(t, fmt.Sprintf("%q is a directory", path), msgAndArgs...) + } + return true +} + +// DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func DirExists(t TestingT, path string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return Fail(t, fmt.Sprintf("unable to find file %q", path), msgAndArgs...) + } + return Fail(t, fmt.Sprintf("error when running os.Lstat(%q): %s", path, err), msgAndArgs...) + } + if !info.IsDir() { + return Fail(t, fmt.Sprintf("%q is a file", path), msgAndArgs...) + } + return true +} + // JSONEq asserts that two JSON strings are equivalent. // // assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -// -// Returns whether the assertion was successful (true) or not (false). func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } var expectedJSONAsInterface, actualJSONAsInterface interface{} if err := json.Unmarshal([]byte(expected), &expectedJSONAsInterface); err != nil { @@ -1206,3 +1380,7 @@ var spewConfig = spew.ConfigState{ DisableCapacities: true, SortKeys: true, } + +type tHelper interface { + Helper() +} diff --git a/vendor/github.com/stretchr/testify/assert/http_assertions.go b/vendor/github.com/stretchr/testify/assert/http_assertions.go index ba811c04dd5f..a87c2e1da043 100644 --- a/vendor/github.com/stretchr/testify/assert/http_assertions.go +++ b/vendor/github.com/stretchr/testify/assert/http_assertions.go @@ -12,10 +12,11 @@ import ( // an error if building a new request fails. func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (int, error) { w := httptest.NewRecorder() - req, err := http.NewRequest(method, url+"?"+values.Encode(), nil) + req, err := http.NewRequest(method, url, nil) if err != nil { return -1, err } + req.URL.RawQuery = values.Encode() handler(w, req) return w.Code, nil } @@ -25,7 +26,10 @@ func httpCode(handler http.HandlerFunc, method, url string, values url.Values) ( // assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). -func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { +func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) @@ -45,7 +49,10 @@ func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, value // assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { +func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) @@ -65,7 +72,10 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, valu // assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values) bool { +func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) @@ -98,7 +108,10 @@ func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) s // assert.HTTPBodyContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool { +func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } body := HTTPBody(handler, method, url, values) contains := strings.Contains(body, fmt.Sprint(str)) @@ -115,7 +128,10 @@ func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, // assert.HTTPBodyNotContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}) bool { +func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } body := HTTPBody(handler, method, url, values) contains := strings.Contains(body, fmt.Sprint(str)) diff --git a/vendor/github.com/stretchr/testify/mock/mock.go b/vendor/github.com/stretchr/testify/mock/mock.go index fc63571d4e8b..4dde47d89d26 100644 --- a/vendor/github.com/stretchr/testify/mock/mock.go +++ b/vendor/github.com/stretchr/testify/mock/mock.go @@ -1,6 +1,7 @@ package mock import ( + "errors" "fmt" "reflect" "regexp" @@ -41,6 +42,9 @@ type Call struct { // this method is called. ReturnArguments Arguments + // Holds the caller info for the On() call + callerInfo []string + // The number of times to return the return arguments when setting // expectations. 0 means to always return the value. Repeatability int @@ -48,22 +52,28 @@ type Call struct { // Amount of times this call has been called totalCalls int + // Call to this method can be optional + optional bool + // Holds a channel that will be used to block the Return until it either // receives a message or is closed. nil means it returns immediately. WaitFor <-chan time.Time + waitTime time.Duration + // Holds a handler used to manipulate arguments content that are passed by // reference. It's useful when mocking methods such as unmarshalers or // decoders. RunFn func(Arguments) } -func newCall(parent *Mock, methodName string, methodArguments ...interface{}) *Call { +func newCall(parent *Mock, methodName string, callerInfo []string, methodArguments ...interface{}) *Call { return &Call{ Parent: parent, Method: methodName, Arguments: methodArguments, ReturnArguments: make([]interface{}, 0), + callerInfo: callerInfo, Repeatability: 0, WaitFor: nil, RunFn: nil, @@ -130,7 +140,10 @@ func (c *Call) WaitUntil(w <-chan time.Time) *Call { // // Mock.On("MyMethod", arg1, arg2).After(time.Second) func (c *Call) After(d time.Duration) *Call { - return c.WaitUntil(time.After(d)) + c.lock() + defer c.unlock() + c.waitTime = d + return c } // Run sets a handler to be called before returning. It can be used when @@ -148,6 +161,15 @@ func (c *Call) Run(fn func(args Arguments)) *Call { return c } +// Maybe allows the method call to be optional. Not calling an optional method +// will not cause an error while asserting expectations +func (c *Call) Maybe() *Call { + c.lock() + defer c.unlock() + c.optional = true + return c +} + // On chains a new expectation description onto the mocked interface. This // allows syntax like. // @@ -169,6 +191,10 @@ type Mock struct { // Holds the calls that were made to this mocked object. Calls []Call + // test is An optional variable that holds the test struct, to be used when an + // invalid mock call was made. + test TestingT + // TestData holds any data that might be useful for testing. Testify ignores // this data completely allowing you to do whatever you like with it. testData objx.Map @@ -191,6 +217,27 @@ func (m *Mock) TestData() objx.Map { Setting expectations */ +// Test sets the test struct variable of the mock object +func (m *Mock) Test(t TestingT) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.test = t +} + +// fail fails the current test with the given formatted format and args. +// In case that a test was defined, it uses the test APIs for failing a test, +// otherwise it uses panic. +func (m *Mock) fail(format string, args ...interface{}) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if m.test == nil { + panic(fmt.Sprintf(format, args...)) + } + m.test.Errorf(format, args...) + m.test.FailNow() +} + // On starts a description of an expectation of the specified method // being called. // @@ -204,7 +251,7 @@ func (m *Mock) On(methodName string, arguments ...interface{}) *Call { m.mutex.Lock() defer m.mutex.Unlock() - c := newCall(m, methodName, arguments...) + c := newCall(m, methodName, assert.CallerInfo(), arguments...) m.ExpectedCalls = append(m.ExpectedCalls, c) return c } @@ -227,27 +274,25 @@ func (m *Mock) findExpectedCall(method string, arguments ...interface{}) (int, * return -1, nil } -func (m *Mock) findClosestCall(method string, arguments ...interface{}) (bool, *Call) { - diffCount := 0 +func (m *Mock) findClosestCall(method string, arguments ...interface{}) (*Call, string) { + var diffCount int var closestCall *Call + var err string for _, call := range m.expectedCalls() { if call.Method == method { - _, tempDiffCount := call.Arguments.Diff(arguments) + errInfo, tempDiffCount := call.Arguments.Diff(arguments) if tempDiffCount < diffCount || diffCount == 0 { diffCount = tempDiffCount closestCall = call + err = errInfo } } } - if closestCall == nil { - return false, nil - } - - return true, closestCall + return closestCall, err } func callString(method string, arguments Arguments, includeArgumentValues bool) string { @@ -294,6 +339,7 @@ func (m *Mock) Called(arguments ...interface{}) Arguments { // If Call.WaitFor is set, blocks until the channel is closed or receives a message. func (m *Mock) MethodCalled(methodName string, arguments ...interface{}) Arguments { m.mutex.Lock() + //TODO: could combine expected and closes in single loop found, call := m.findExpectedCall(methodName, arguments...) if found < 0 { @@ -304,43 +350,52 @@ func (m *Mock) MethodCalled(methodName string, arguments ...interface{}) Argumen // b) the arguments are not what was expected, or // c) the developer has forgotten to add an accompanying On...Return pair. - closestFound, closestCall := m.findClosestCall(methodName, arguments...) + closestCall, mismatch := m.findClosestCall(methodName, arguments...) m.mutex.Unlock() - if closestFound { - panic(fmt.Sprintf("\n\nmock: Unexpected Method Call\n-----------------------------\n\n%s\n\nThe closest call I have is: \n\n%s\n\n%s\n", callString(methodName, arguments, true), callString(methodName, closestCall.Arguments, true), diffArguments(arguments, closestCall.Arguments))) + if closestCall != nil { + m.fail("\n\nmock: Unexpected Method Call\n-----------------------------\n\n%s\n\nThe closest call I have is: \n\n%s\n\n%s\nDiff: %s", + callString(methodName, arguments, true), + callString(methodName, closestCall.Arguments, true), + diffArguments(closestCall.Arguments, arguments), + strings.TrimSpace(mismatch), + ) } else { - panic(fmt.Sprintf("\nassert: mock: I don't know what to return because the method call was unexpected.\n\tEither do Mock.On(\"%s\").Return(...) first, or remove the %s() call.\n\tThis method was unexpected:\n\t\t%s\n\tat: %s", methodName, methodName, callString(methodName, arguments, true), assert.CallerInfo())) + m.fail("\nassert: mock: I don't know what to return because the method call was unexpected.\n\tEither do Mock.On(\"%s\").Return(...) first, or remove the %s() call.\n\tThis method was unexpected:\n\t\t%s\n\tat: %s", methodName, methodName, callString(methodName, arguments, true), assert.CallerInfo()) } } - switch { - case call.Repeatability == 1: + if call.Repeatability == 1 { call.Repeatability = -1 - call.totalCalls++ - - case call.Repeatability > 1: + } else if call.Repeatability > 1 { call.Repeatability-- - call.totalCalls++ - - case call.Repeatability == 0: - call.totalCalls++ } + call.totalCalls++ // add the call - m.Calls = append(m.Calls, *newCall(m, methodName, arguments...)) + m.Calls = append(m.Calls, *newCall(m, methodName, assert.CallerInfo(), arguments...)) m.mutex.Unlock() // block if specified if call.WaitFor != nil { <-call.WaitFor + } else { + time.Sleep(call.waitTime) } - if call.RunFn != nil { - call.RunFn(arguments) + m.mutex.Lock() + runFn := call.RunFn + m.mutex.Unlock() + + if runFn != nil { + runFn(arguments) } - return call.ReturnArguments + m.mutex.Lock() + returnArgs := call.ReturnArguments + m.mutex.Unlock() + + return returnArgs } /* @@ -356,6 +411,9 @@ type assertExpectationser interface { // // Calls may have occurred in any order. func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } for _, obj := range testObjects { if m, ok := obj.(Mock); ok { t.Logf("Deprecated mock.AssertExpectationsForObjects(myMock.Mock) use mock.AssertExpectationsForObjects(myMock)") @@ -363,6 +421,7 @@ func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool { } m := obj.(assertExpectationser) if !m.AssertExpectations(t) { + t.Logf("Expectations didn't match for Mock: %+v", reflect.TypeOf(m)) return false } } @@ -372,6 +431,9 @@ func AssertExpectationsForObjects(t TestingT, testObjects ...interface{}) bool { // AssertExpectations asserts that everything specified with On and Return was // in fact called as expected. Calls may have occurred in any order. func (m *Mock) AssertExpectations(t TestingT) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } m.mutex.Lock() defer m.mutex.Unlock() var somethingMissing bool @@ -380,16 +442,17 @@ func (m *Mock) AssertExpectations(t TestingT) bool { // iterate through each expectation expectedCalls := m.expectedCalls() for _, expectedCall := range expectedCalls { - if !m.methodWasCalled(expectedCall.Method, expectedCall.Arguments) && expectedCall.totalCalls == 0 { + if !expectedCall.optional && !m.methodWasCalled(expectedCall.Method, expectedCall.Arguments) && expectedCall.totalCalls == 0 { somethingMissing = true failedExpectations++ - t.Logf("\u274C\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String()) + t.Logf("FAIL:\t%s(%s)\n\t\tat: %s", expectedCall.Method, expectedCall.Arguments.String(), expectedCall.callerInfo) } else { if expectedCall.Repeatability > 0 { somethingMissing = true failedExpectations++ + t.Logf("FAIL:\t%s(%s)\n\t\tat: %s", expectedCall.Method, expectedCall.Arguments.String(), expectedCall.callerInfo) } else { - t.Logf("\u2705\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String()) + t.Logf("PASS:\t%s(%s)", expectedCall.Method, expectedCall.Arguments.String()) } } } @@ -403,6 +466,9 @@ func (m *Mock) AssertExpectations(t TestingT) bool { // AssertNumberOfCalls asserts that the method was called expectedCalls times. func (m *Mock) AssertNumberOfCalls(t TestingT, methodName string, expectedCalls int) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } m.mutex.Lock() defer m.mutex.Unlock() var actualCalls int @@ -417,11 +483,22 @@ func (m *Mock) AssertNumberOfCalls(t TestingT, methodName string, expectedCalls // AssertCalled asserts that the method was called. // It can produce a false result when an argument is a pointer type and the underlying value changed after calling the mocked method. func (m *Mock) AssertCalled(t TestingT, methodName string, arguments ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } m.mutex.Lock() defer m.mutex.Unlock() - if !assert.True(t, m.methodWasCalled(methodName, arguments), fmt.Sprintf("The \"%s\" method should have been called with %d argument(s), but was not.", methodName, len(arguments))) { - t.Logf("%v", m.expectedCalls()) - return false + if !m.methodWasCalled(methodName, arguments) { + var calledWithArgs []string + for _, call := range m.calls() { + calledWithArgs = append(calledWithArgs, fmt.Sprintf("%v", call.Arguments)) + } + if len(calledWithArgs) == 0 { + return assert.Fail(t, "Should have called with given arguments", + fmt.Sprintf("Expected %q to have been called with:\n%v\nbut no actual calls happened", methodName, arguments)) + } + return assert.Fail(t, "Should have called with given arguments", + fmt.Sprintf("Expected %q to have been called with:\n%v\nbut actual calls were:\n %v", methodName, arguments, strings.Join(calledWithArgs, "\n"))) } return true } @@ -429,11 +506,14 @@ func (m *Mock) AssertCalled(t TestingT, methodName string, arguments ...interfac // AssertNotCalled asserts that the method was not called. // It can produce a false result when an argument is a pointer type and the underlying value changed after calling the mocked method. func (m *Mock) AssertNotCalled(t TestingT, methodName string, arguments ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } m.mutex.Lock() defer m.mutex.Unlock() - if !assert.False(t, m.methodWasCalled(methodName, arguments), fmt.Sprintf("The \"%s\" method was called with %d argument(s), but should NOT have been.", methodName, len(arguments))) { - t.Logf("%v", m.expectedCalls()) - return false + if m.methodWasCalled(methodName, arguments) { + return assert.Fail(t, "Should not have called with given arguments", + fmt.Sprintf("Expected %q to not have been called with:\n%v\nbut actually it was.", methodName, arguments)) } return true } @@ -473,7 +553,7 @@ type Arguments []interface{} const ( // Anything is used in Diff and Assert when the argument being tested // shouldn't be taken into consideration. - Anything string = "mock.Anything" + Anything = "mock.Anything" ) // AnythingOfTypeArgument is a string that contains the type of an argument @@ -498,9 +578,25 @@ type argumentMatcher struct { func (f argumentMatcher) Matches(argument interface{}) bool { expectType := f.fn.Type().In(0) + expectTypeNilSupported := false + switch expectType.Kind() { + case reflect.Interface, reflect.Chan, reflect.Func, reflect.Map, reflect.Slice, reflect.Ptr: + expectTypeNilSupported = true + } - if reflect.TypeOf(argument).AssignableTo(expectType) { - result := f.fn.Call([]reflect.Value{reflect.ValueOf(argument)}) + argType := reflect.TypeOf(argument) + var arg reflect.Value + if argType == nil { + arg = reflect.New(expectType).Elem() + } else { + arg = reflect.ValueOf(argument) + } + + if argType == nil && !expectTypeNilSupported { + panic(errors.New("attempting to call matcher with nil for non-nil expected type")) + } + if argType == nil || argType.AssignableTo(expectType) { + result := f.fn.Call([]reflect.Value{arg}) return result[0].Bool() } return false @@ -560,6 +656,7 @@ func (args Arguments) Is(objects ...interface{}) bool { // // Returns the diff string and number of differences found. func (args Arguments) Diff(objects []interface{}) (string, int) { + //TODO: could return string as error and nil for No difference var output = "\n" var differences int @@ -586,10 +683,10 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { if matcher, ok := expected.(argumentMatcher); ok { if matcher.Matches(actual) { - output = fmt.Sprintf("%s\t%d: \u2705 %s matched by %s\n", output, i, actual, matcher) + output = fmt.Sprintf("%s\t%d: PASS: %s matched by %s\n", output, i, actual, matcher) } else { differences++ - output = fmt.Sprintf("%s\t%d: \u2705 %s not matched by %s\n", output, i, actual, matcher) + output = fmt.Sprintf("%s\t%d: PASS: %s not matched by %s\n", output, i, actual, matcher) } } else if reflect.TypeOf(expected) == reflect.TypeOf((*AnythingOfTypeArgument)(nil)).Elem() { @@ -597,7 +694,7 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { if reflect.TypeOf(actual).Name() != string(expected.(AnythingOfTypeArgument)) && reflect.TypeOf(actual).String() != string(expected.(AnythingOfTypeArgument)) { // not match differences++ - output = fmt.Sprintf("%s\t%d: \u274C type %s != type %s - %s\n", output, i, expected, reflect.TypeOf(actual).Name(), actual) + output = fmt.Sprintf("%s\t%d: FAIL: type %s != type %s - %s\n", output, i, expected, reflect.TypeOf(actual).Name(), actual) } } else { @@ -606,11 +703,11 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { if assert.ObjectsAreEqual(expected, Anything) || assert.ObjectsAreEqual(actual, Anything) || assert.ObjectsAreEqual(actual, expected) { // match - output = fmt.Sprintf("%s\t%d: \u2705 %s == %s\n", output, i, actual, expected) + output = fmt.Sprintf("%s\t%d: PASS: %s == %s\n", output, i, actual, expected) } else { // not match differences++ - output = fmt.Sprintf("%s\t%d: \u274C %s != %s\n", output, i, actual, expected) + output = fmt.Sprintf("%s\t%d: FAIL: %s != %s\n", output, i, actual, expected) } } @@ -627,6 +724,9 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { // Assert compares the arguments with the specified objects and fails if // they do not exactly match. func (args Arguments) Assert(t TestingT, objects ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } // get the differences diff, diffCount := args.Diff(objects) @@ -774,3 +874,7 @@ var spewConfig = spew.ConfigState{ DisableCapacities: true, SortKeys: true, } + +type tHelper interface { + Helper() +} diff --git a/vendor/github.com/stretchr/testify/require/require.go b/vendor/github.com/stretchr/testify/require/require.go index 2fe055784644..2110ea89c1cc 100644 --- a/vendor/github.com/stretchr/testify/require/require.go +++ b/vendor/github.com/stretchr/testify/require/require.go @@ -14,6 +14,9 @@ import ( // Condition uses a Comparison to assert a complex condition. func Condition(t TestingT, comp assert.Comparison, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Condition(t, comp, msgAndArgs...) { t.FailNow() } @@ -21,6 +24,9 @@ func Condition(t TestingT, comp assert.Comparison, msgAndArgs ...interface{}) { // Conditionf uses a Comparison to assert a complex condition. func Conditionf(t TestingT, comp assert.Comparison, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Conditionf(t, comp, msg, args...) { t.FailNow() } @@ -32,9 +38,10 @@ func Conditionf(t TestingT, comp assert.Comparison, msg string, args ...interfac // assert.Contains(t, "Hello World", "World") // assert.Contains(t, ["Hello", "World"], "World") // assert.Contains(t, {"Hello": "World"}, "Hello") -// -// Returns whether the assertion was successful (true) or not (false). func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Contains(t, s, contains, msgAndArgs...) { t.FailNow() } @@ -46,21 +53,71 @@ func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...int // assert.Containsf(t, "Hello World", "World", "error message %s", "formatted") // assert.Containsf(t, ["Hello", "World"], "World", "error message %s", "formatted") // assert.Containsf(t, {"Hello": "World"}, "Hello", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Containsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Containsf(t, s, contains, msg, args...) { t.FailNow() } } +// DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func DirExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.DirExists(t, path, msgAndArgs...) { + t.FailNow() + } +} + +// DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func DirExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.DirExistsf(t, path, msg, args...) { + t.FailNow() + } +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatch(t, [1, 3, 2, 3], [1, 3, 3, 2]) +func ElementsMatch(t TestingT, listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.ElementsMatch(t, listA, listB, msgAndArgs...) { + t.FailNow() + } +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// assert.ElementsMatchf(t, [1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.ElementsMatchf(t, listA, listB, msg, args...) { + t.FailNow() + } +} + // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // assert.Empty(t, obj) -// -// Returns whether the assertion was successful (true) or not (false). func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Empty(t, object, msgAndArgs...) { t.FailNow() } @@ -70,9 +127,10 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { // a slice or a channel with len == 0. // // assert.Emptyf(t, obj, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Emptyf(t, object, msg, args...) { t.FailNow() } @@ -82,12 +140,13 @@ func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { // // assert.Equal(t, 123, 123) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Equal(t, expected, actual, msgAndArgs...) { t.FailNow() } @@ -98,9 +157,10 @@ func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...i // // actualObj, err := SomeFunction() // assert.EqualError(t, err, expectedErrorString) -// -// Returns whether the assertion was successful (true) or not (false). func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.EqualError(t, theError, errString, msgAndArgs...) { t.FailNow() } @@ -111,9 +171,10 @@ func EqualError(t TestingT, theError error, errString string, msgAndArgs ...inte // // actualObj, err := SomeFunction() // assert.EqualErrorf(t, err, expectedErrorString, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func EqualErrorf(t TestingT, theError error, errString string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.EqualErrorf(t, theError, errString, msg, args...) { t.FailNow() } @@ -123,9 +184,10 @@ func EqualErrorf(t TestingT, theError error, errString string, msg string, args // and equal. // // assert.EqualValues(t, uint32(123), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.EqualValues(t, expected, actual, msgAndArgs...) { t.FailNow() } @@ -135,9 +197,10 @@ func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArg // and equal. // // assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.EqualValuesf(t, expected, actual, msg, args...) { t.FailNow() } @@ -147,12 +210,13 @@ func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg stri // // assert.Equalf(t, 123, 123, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Equalf(t, expected, actual, msg, args...) { t.FailNow() } @@ -164,9 +228,10 @@ func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, ar // if assert.Error(t, err) { // assert.Equal(t, expectedError, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func Error(t TestingT, err error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Error(t, err, msgAndArgs...) { t.FailNow() } @@ -178,31 +243,34 @@ func Error(t TestingT, err error, msgAndArgs ...interface{}) { // if assert.Errorf(t, err, "error message %s", "formatted") { // assert.Equal(t, expectedErrorf, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func Errorf(t TestingT, err error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Errorf(t, err, msg, args...) { t.FailNow() } } -// Exactly asserts that two objects are equal is value and type. +// Exactly asserts that two objects are equal in value and type. // // assert.Exactly(t, int32(123), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Exactly(t, expected, actual, msgAndArgs...) { t.FailNow() } } -// Exactlyf asserts that two objects are equal is value and type. +// Exactlyf asserts that two objects are equal in value and type. // // assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Exactlyf(t, expected, actual, msg, args...) { t.FailNow() } @@ -210,6 +278,9 @@ func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, // Fail reports a failure through func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Fail(t, failureMessage, msgAndArgs...) { t.FailNow() } @@ -217,6 +288,9 @@ func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) { // FailNow fails test func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.FailNow(t, failureMessage, msgAndArgs...) { t.FailNow() } @@ -224,6 +298,9 @@ func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) { // FailNowf fails test func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.FailNowf(t, failureMessage, msg, args...) { t.FailNow() } @@ -231,6 +308,9 @@ func FailNowf(t TestingT, failureMessage string, msg string, args ...interface{} // Failf reports a failure through func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Failf(t, failureMessage, msg, args...) { t.FailNow() } @@ -239,9 +319,10 @@ func Failf(t TestingT, failureMessage string, msg string, args ...interface{}) { // False asserts that the specified value is false. // // assert.False(t, myBool) -// -// Returns whether the assertion was successful (true) or not (false). func False(t TestingT, value bool, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.False(t, value, msgAndArgs...) { t.FailNow() } @@ -250,22 +331,46 @@ func False(t TestingT, value bool, msgAndArgs ...interface{}) { // Falsef asserts that the specified value is false. // // assert.Falsef(t, myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Falsef(t TestingT, value bool, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Falsef(t, value, msg, args...) { t.FailNow() } } +// FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func FileExists(t TestingT, path string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.FileExists(t, path, msgAndArgs...) { + t.FailNow() + } +} + +// FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func FileExistsf(t TestingT, path string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.FileExistsf(t, path, msg, args...) { + t.FailNow() + } +} + // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // // assert.HTTPBodyContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - if !assert.HTTPBodyContains(t, handler, method, url, values, str) { +func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPBodyContains(t, handler, method, url, values, str, msgAndArgs...) { t.FailNow() } } @@ -276,8 +381,11 @@ func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url s // assert.HTTPBodyContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - if !assert.HTTPBodyContainsf(t, handler, method, url, values, str) { +func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPBodyContainsf(t, handler, method, url, values, str, msg, args...) { t.FailNow() } } @@ -288,8 +396,11 @@ func HTTPBodyContainsf(t TestingT, handler http.HandlerFunc, method string, url // assert.HTTPBodyNotContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - if !assert.HTTPBodyNotContains(t, handler, method, url, values, str) { +func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPBodyNotContains(t, handler, method, url, values, str, msgAndArgs...) { t.FailNow() } } @@ -300,8 +411,11 @@ func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, ur // assert.HTTPBodyNotContainsf(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - if !assert.HTTPBodyNotContainsf(t, handler, method, url, values, str) { +func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPBodyNotContainsf(t, handler, method, url, values, str, msg, args...) { t.FailNow() } } @@ -311,8 +425,11 @@ func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, u // assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPError(t, handler, method, url, values) { +func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPError(t, handler, method, url, values, msgAndArgs...) { t.FailNow() } } @@ -322,8 +439,11 @@ func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, // assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPErrorf(t, handler, method, url, values) { +func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPErrorf(t, handler, method, url, values, msg, args...) { t.FailNow() } } @@ -333,8 +453,11 @@ func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, // assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPRedirect(t, handler, method, url, values) { +func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPRedirect(t, handler, method, url, values, msgAndArgs...) { t.FailNow() } } @@ -344,8 +467,11 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url strin // assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPRedirectf(t, handler, method, url, values) { +func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPRedirectf(t, handler, method, url, values, msg, args...) { t.FailNow() } } @@ -355,8 +481,11 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri // assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). -func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPSuccess(t, handler, method, url, values) { +func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPSuccess(t, handler, method, url, values, msgAndArgs...) { t.FailNow() } } @@ -366,8 +495,11 @@ func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string // assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { - if !assert.HTTPSuccessf(t, handler, method, url, values) { +func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.HTTPSuccessf(t, handler, method, url, values, msg, args...) { t.FailNow() } } @@ -376,6 +508,9 @@ func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url strin // // assert.Implements(t, (*MyInterface)(nil), new(MyObject)) func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Implements(t, interfaceObject, object, msgAndArgs...) { t.FailNow() } @@ -385,6 +520,9 @@ func Implements(t TestingT, interfaceObject interface{}, object interface{}, msg // // assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Implementsf(t, interfaceObject, object, msg, args...) { t.FailNow() } @@ -393,16 +531,40 @@ func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, ms // InDelta asserts that the two numerals are within delta of each other. // // assert.InDelta(t, math.Pi, (22 / 7.0), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func InDelta(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InDelta(t, expected, actual, delta, msgAndArgs...) { t.FailNow() } } +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValues(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.InDeltaMapValues(t, expected, actual, delta, msgAndArgs...) { + t.FailNow() + } +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func InDeltaMapValuesf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if !assert.InDeltaMapValuesf(t, expected, actual, delta, msg, args...) { + t.FailNow() + } +} + // InDeltaSlice is the same as InDelta, except it compares two slices. func InDeltaSlice(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InDeltaSlice(t, expected, actual, delta, msgAndArgs...) { t.FailNow() } @@ -410,6 +572,9 @@ func InDeltaSlice(t TestingT, expected interface{}, actual interface{}, delta fl // InDeltaSlicef is the same as InDelta, except it compares two slices. func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InDeltaSlicef(t, expected, actual, delta, msg, args...) { t.FailNow() } @@ -418,18 +583,20 @@ func InDeltaSlicef(t TestingT, expected interface{}, actual interface{}, delta f // InDeltaf asserts that the two numerals are within delta of each other. // // assert.InDeltaf(t, math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func InDeltaf(t TestingT, expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InDeltaf(t, expected, actual, delta, msg, args...) { t.FailNow() } } // InEpsilon asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func InEpsilon(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InEpsilon(t, expected, actual, epsilon, msgAndArgs...) { t.FailNow() } @@ -437,6 +604,9 @@ func InEpsilon(t TestingT, expected interface{}, actual interface{}, epsilon flo // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlice(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InEpsilonSlice(t, expected, actual, epsilon, msgAndArgs...) { t.FailNow() } @@ -444,15 +614,19 @@ func InEpsilonSlice(t TestingT, expected interface{}, actual interface{}, epsilo // InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. func InEpsilonSlicef(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InEpsilonSlicef(t, expected, actual, epsilon, msg, args...) { t.FailNow() } } // InEpsilonf asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.InEpsilonf(t, expected, actual, epsilon, msg, args...) { t.FailNow() } @@ -460,6 +634,9 @@ func InEpsilonf(t TestingT, expected interface{}, actual interface{}, epsilon fl // IsType asserts that the specified objects are of the same type. func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.IsType(t, expectedType, object, msgAndArgs...) { t.FailNow() } @@ -467,6 +644,9 @@ func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs // IsTypef asserts that the specified objects are of the same type. func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.IsTypef(t, expectedType, object, msg, args...) { t.FailNow() } @@ -475,9 +655,10 @@ func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg strin // JSONEq asserts that two JSON strings are equivalent. // // assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -// -// Returns whether the assertion was successful (true) or not (false). func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.JSONEq(t, expected, actual, msgAndArgs...) { t.FailNow() } @@ -486,9 +667,10 @@ func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{ // JSONEqf asserts that two JSON strings are equivalent. // // assert.JSONEqf(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func JSONEqf(t TestingT, expected string, actual string, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.JSONEqf(t, expected, actual, msg, args...) { t.FailNow() } @@ -498,9 +680,10 @@ func JSONEqf(t TestingT, expected string, actual string, msg string, args ...int // Len also fails if the object has a type that len() not accept. // // assert.Len(t, mySlice, 3) -// -// Returns whether the assertion was successful (true) or not (false). func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Len(t, object, length, msgAndArgs...) { t.FailNow() } @@ -510,9 +693,10 @@ func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) // Lenf also fails if the object has a type that len() not accept. // // assert.Lenf(t, mySlice, 3, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Lenf(t TestingT, object interface{}, length int, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Lenf(t, object, length, msg, args...) { t.FailNow() } @@ -521,9 +705,10 @@ func Lenf(t TestingT, object interface{}, length int, msg string, args ...interf // Nil asserts that the specified object is nil. // // assert.Nil(t, err) -// -// Returns whether the assertion was successful (true) or not (false). func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Nil(t, object, msgAndArgs...) { t.FailNow() } @@ -532,9 +717,10 @@ func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { // Nilf asserts that the specified object is nil. // // assert.Nilf(t, err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Nilf(t, object, msg, args...) { t.FailNow() } @@ -546,9 +732,10 @@ func Nilf(t TestingT, object interface{}, msg string, args ...interface{}) { // if assert.NoError(t, err) { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func NoError(t TestingT, err error, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NoError(t, err, msgAndArgs...) { t.FailNow() } @@ -560,9 +747,10 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) { // if assert.NoErrorf(t, err, "error message %s", "formatted") { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func NoErrorf(t TestingT, err error, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NoErrorf(t, err, msg, args...) { t.FailNow() } @@ -574,9 +762,10 @@ func NoErrorf(t TestingT, err error, msg string, args ...interface{}) { // assert.NotContains(t, "Hello World", "Earth") // assert.NotContains(t, ["Hello", "World"], "Earth") // assert.NotContains(t, {"Hello": "World"}, "Earth") -// -// Returns whether the assertion was successful (true) or not (false). func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotContains(t, s, contains, msgAndArgs...) { t.FailNow() } @@ -588,9 +777,10 @@ func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ... // assert.NotContainsf(t, "Hello World", "Earth", "error message %s", "formatted") // assert.NotContainsf(t, ["Hello", "World"], "Earth", "error message %s", "formatted") // assert.NotContainsf(t, {"Hello": "World"}, "Earth", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotContainsf(t, s, contains, msg, args...) { t.FailNow() } @@ -602,9 +792,10 @@ func NotContainsf(t TestingT, s interface{}, contains interface{}, msg string, a // if assert.NotEmpty(t, obj) { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotEmpty(t, object, msgAndArgs...) { t.FailNow() } @@ -616,9 +807,10 @@ func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { // if assert.NotEmptyf(t, obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotEmptyf(t, object, msg, args...) { t.FailNow() } @@ -628,11 +820,12 @@ func NotEmptyf(t TestingT, object interface{}, msg string, args ...interface{}) // // assert.NotEqual(t, obj1, obj2) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotEqual(t, expected, actual, msgAndArgs...) { t.FailNow() } @@ -642,11 +835,12 @@ func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs . // // assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotEqualf(t, expected, actual, msg, args...) { t.FailNow() } @@ -655,9 +849,10 @@ func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, // NotNil asserts that the specified object is not nil. // // assert.NotNil(t, err) -// -// Returns whether the assertion was successful (true) or not (false). func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotNil(t, object, msgAndArgs...) { t.FailNow() } @@ -666,9 +861,10 @@ func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { // NotNilf asserts that the specified object is not nil. // // assert.NotNilf(t, err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotNilf(t, object, msg, args...) { t.FailNow() } @@ -677,9 +873,10 @@ func NotNilf(t TestingT, object interface{}, msg string, args ...interface{}) { // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // assert.NotPanics(t, func(){ RemainCalm() }) -// -// Returns whether the assertion was successful (true) or not (false). func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotPanics(t, f, msgAndArgs...) { t.FailNow() } @@ -688,9 +885,10 @@ func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { // NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. // // assert.NotPanicsf(t, func(){ RemainCalm() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotPanicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotPanicsf(t, f, msg, args...) { t.FailNow() } @@ -700,9 +898,10 @@ func NotPanicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interfac // // assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") // assert.NotRegexp(t, "^start", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotRegexp(t, rx, str, msgAndArgs...) { t.FailNow() } @@ -712,9 +911,10 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf // // assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") // assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotRegexpf(t, rx, str, msg, args...) { t.FailNow() } @@ -724,9 +924,10 @@ func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args .. // elements given in the specified subset(array, slice...). // // assert.NotSubset(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotSubset(t, list, subset, msgAndArgs...) { t.FailNow() } @@ -736,23 +937,30 @@ func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...i // elements given in the specified subset(array, slice...). // // assert.NotSubsetf(t, [1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotSubsetf(t, list, subset, msg, args...) { t.FailNow() } } -// NotZero asserts that i is not the zero value for its type and returns the truth. +// NotZero asserts that i is not the zero value for its type. func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotZero(t, i, msgAndArgs...) { t.FailNow() } } -// NotZerof asserts that i is not the zero value for its type and returns the truth. +// NotZerof asserts that i is not the zero value for its type. func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.NotZerof(t, i, msg, args...) { t.FailNow() } @@ -761,9 +969,10 @@ func NotZerof(t TestingT, i interface{}, msg string, args ...interface{}) { // Panics asserts that the code inside the specified PanicTestFunc panics. // // assert.Panics(t, func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Panics(t, f, msgAndArgs...) { t.FailNow() } @@ -773,9 +982,10 @@ func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { // the recovered panic value equals the expected panic value. // // assert.PanicsWithValue(t, "crazy error", func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func PanicsWithValue(t TestingT, expected interface{}, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.PanicsWithValue(t, expected, f, msgAndArgs...) { t.FailNow() } @@ -785,9 +995,10 @@ func PanicsWithValue(t TestingT, expected interface{}, f assert.PanicTestFunc, m // the recovered panic value equals the expected panic value. // // assert.PanicsWithValuef(t, "crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func PanicsWithValuef(t TestingT, expected interface{}, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.PanicsWithValuef(t, expected, f, msg, args...) { t.FailNow() } @@ -796,9 +1007,10 @@ func PanicsWithValuef(t TestingT, expected interface{}, f assert.PanicTestFunc, // Panicsf asserts that the code inside the specified PanicTestFunc panics. // // assert.Panicsf(t, func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Panicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Panicsf(t, f, msg, args...) { t.FailNow() } @@ -808,9 +1020,10 @@ func Panicsf(t TestingT, f assert.PanicTestFunc, msg string, args ...interface{} // // assert.Regexp(t, regexp.MustCompile("start"), "it's starting") // assert.Regexp(t, "start...$", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Regexp(t, rx, str, msgAndArgs...) { t.FailNow() } @@ -820,9 +1033,10 @@ func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface // // assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") // assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Regexpf(t, rx, str, msg, args...) { t.FailNow() } @@ -832,9 +1046,10 @@ func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...in // elements given in the specified subset(array, slice...). // // assert.Subset(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Subset(t, list, subset, msgAndArgs...) { t.FailNow() } @@ -844,9 +1059,10 @@ func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...inte // elements given in the specified subset(array, slice...). // // assert.Subsetf(t, [1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Subsetf(t, list, subset, msg, args...) { t.FailNow() } @@ -855,9 +1071,10 @@ func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args // True asserts that the specified value is true. // // assert.True(t, myBool) -// -// Returns whether the assertion was successful (true) or not (false). func True(t TestingT, value bool, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.True(t, value, msgAndArgs...) { t.FailNow() } @@ -866,9 +1083,10 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) { // Truef asserts that the specified value is true. // // assert.Truef(t, myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func Truef(t TestingT, value bool, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Truef(t, value, msg, args...) { t.FailNow() } @@ -877,9 +1095,10 @@ func Truef(t TestingT, value bool, msg string, args ...interface{}) { // WithinDuration asserts that the two times are within duration delta of each other. // // assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second) -// -// Returns whether the assertion was successful (true) or not (false). func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.WithinDuration(t, expected, actual, delta, msgAndArgs...) { t.FailNow() } @@ -888,23 +1107,30 @@ func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time // WithinDurationf asserts that the two times are within duration delta of each other. // // assert.WithinDurationf(t, time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func WithinDurationf(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.WithinDurationf(t, expected, actual, delta, msg, args...) { t.FailNow() } } -// Zero asserts that i is the zero value for its type and returns the truth. +// Zero asserts that i is the zero value for its type. func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Zero(t, i, msgAndArgs...) { t.FailNow() } } -// Zerof asserts that i is the zero value for its type and returns the truth. +// Zerof asserts that i is the zero value for its type. func Zerof(t TestingT, i interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } if !assert.Zerof(t, i, msg, args...) { t.FailNow() } diff --git a/vendor/github.com/stretchr/testify/require/require.go.tmpl b/vendor/github.com/stretchr/testify/require/require.go.tmpl index d2c38f6f2864..9fb2577d4043 100644 --- a/vendor/github.com/stretchr/testify/require/require.go.tmpl +++ b/vendor/github.com/stretchr/testify/require/require.go.tmpl @@ -1,5 +1,6 @@ {{.Comment}} func {{.DocInfo.Name}}(t TestingT, {{.Params}}) { + if h, ok := t.(tHelper); ok { h.Helper() } if !assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) { t.FailNow() } diff --git a/vendor/github.com/stretchr/testify/require/require_forward.go b/vendor/github.com/stretchr/testify/require/require_forward.go index c59c3c7b475e..9c66cd3c329b 100644 --- a/vendor/github.com/stretchr/testify/require/require_forward.go +++ b/vendor/github.com/stretchr/testify/require/require_forward.go @@ -14,11 +14,17 @@ import ( // Condition uses a Comparison to assert a complex condition. func (a *Assertions) Condition(comp assert.Comparison, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Condition(a.t, comp, msgAndArgs...) } // Conditionf uses a Comparison to assert a complex condition. func (a *Assertions) Conditionf(comp assert.Comparison, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Conditionf(a.t, comp, msg, args...) } @@ -28,9 +34,10 @@ func (a *Assertions) Conditionf(comp assert.Comparison, msg string, args ...inte // a.Contains("Hello World", "World") // a.Contains(["Hello", "World"], "World") // a.Contains({"Hello": "World"}, "Hello") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Contains(a.t, s, contains, msgAndArgs...) } @@ -40,19 +47,61 @@ func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs .. // a.Containsf("Hello World", "World", "error message %s", "formatted") // a.Containsf(["Hello", "World"], "World", "error message %s", "formatted") // a.Containsf({"Hello": "World"}, "Hello", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Containsf(s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Containsf(a.t, s, contains, msg, args...) } +// DirExists checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + DirExists(a.t, path, msgAndArgs...) +} + +// DirExistsf checks whether a directory exists in the given path. It also fails if the path is a file rather a directory or there is an error checking whether it exists. +func (a *Assertions) DirExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + DirExistsf(a.t, path, msg, args...) +} + +// ElementsMatch asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatch([1, 3, 2, 3], [1, 3, 3, 2]) +func (a *Assertions) ElementsMatch(listA interface{}, listB interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ElementsMatch(a.t, listA, listB, msgAndArgs...) +} + +// ElementsMatchf asserts that the specified listA(array, slice...) is equal to specified +// listB(array, slice...) ignoring the order of the elements. If there are duplicate elements, +// the number of appearances of each of them in both lists should match. +// +// a.ElementsMatchf([1, 3, 2, 3], [1, 3, 3, 2], "error message %s", "formatted") +func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + ElementsMatchf(a.t, listA, listB, msg, args...) +} + // Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either // a slice or a channel with len == 0. // // a.Empty(obj) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Empty(a.t, object, msgAndArgs...) } @@ -60,9 +109,10 @@ func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { // a slice or a channel with len == 0. // // a.Emptyf(obj, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Emptyf(a.t, object, msg, args...) } @@ -70,12 +120,13 @@ func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) // // a.Equal(123, 123) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Equal(a.t, expected, actual, msgAndArgs...) } @@ -84,9 +135,10 @@ func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs // // actualObj, err := SomeFunction() // a.EqualError(err, expectedErrorString) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } EqualError(a.t, theError, errString, msgAndArgs...) } @@ -95,9 +147,10 @@ func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ... // // actualObj, err := SomeFunction() // a.EqualErrorf(err, expectedErrorString, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualErrorf(theError error, errString string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } EqualErrorf(a.t, theError, errString, msg, args...) } @@ -105,9 +158,10 @@ func (a *Assertions) EqualErrorf(theError error, errString string, msg string, a // and equal. // // a.EqualValues(uint32(123), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } EqualValues(a.t, expected, actual, msgAndArgs...) } @@ -115,9 +169,10 @@ func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAn // and equal. // // a.EqualValuesf(uint32(123, "error message %s", "formatted"), int32(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } EqualValuesf(a.t, expected, actual, msg, args...) } @@ -125,12 +180,13 @@ func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg // // a.Equalf(123, 123, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). Function equality // cannot be determined and will always fail. func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Equalf(a.t, expected, actual, msg, args...) } @@ -140,9 +196,10 @@ func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string // if a.Error(err) { // assert.Equal(t, expectedError, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Error(err error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Error(a.t, err, msgAndArgs...) } @@ -152,76 +209,112 @@ func (a *Assertions) Error(err error, msgAndArgs ...interface{}) { // if a.Errorf(err, "error message %s", "formatted") { // assert.Equal(t, expectedErrorf, err) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Errorf(err error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Errorf(a.t, err, msg, args...) } -// Exactly asserts that two objects are equal is value and type. +// Exactly asserts that two objects are equal in value and type. // // a.Exactly(int32(123), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Exactly(a.t, expected, actual, msgAndArgs...) } -// Exactlyf asserts that two objects are equal is value and type. +// Exactlyf asserts that two objects are equal in value and type. // // a.Exactlyf(int32(123, "error message %s", "formatted"), int64(123)) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Exactlyf(a.t, expected, actual, msg, args...) } // Fail reports a failure through func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Fail(a.t, failureMessage, msgAndArgs...) } // FailNow fails test func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } FailNow(a.t, failureMessage, msgAndArgs...) } // FailNowf fails test func (a *Assertions) FailNowf(failureMessage string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } FailNowf(a.t, failureMessage, msg, args...) } // Failf reports a failure through func (a *Assertions) Failf(failureMessage string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Failf(a.t, failureMessage, msg, args...) } // False asserts that the specified value is false. // // a.False(myBool) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) False(value bool, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } False(a.t, value, msgAndArgs...) } // Falsef asserts that the specified value is false. // // a.Falsef(myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Falsef(value bool, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Falsef(a.t, value, msg, args...) } +// FileExists checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExists(path string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FileExists(a.t, path, msgAndArgs...) +} + +// FileExistsf checks whether a file exists in the given path. It also fails if the path points to a directory or there is an error when trying to check the file. +func (a *Assertions) FileExistsf(path string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + FileExistsf(a.t, path, msg, args...) +} + // HTTPBodyContains asserts that a specified handler returns a // body that contains a string. // // a.HTTPBodyContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - HTTPBodyContains(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyContains(a.t, handler, method, url, values, str, msgAndArgs...) } // HTTPBodyContainsf asserts that a specified handler returns a @@ -230,8 +323,11 @@ func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, u // a.HTTPBodyContainsf(myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - HTTPBodyContainsf(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyContainsf(a.t, handler, method, url, values, str, msg, args...) } // HTTPBodyNotContains asserts that a specified handler returns a @@ -240,8 +336,11 @@ func (a *Assertions) HTTPBodyContainsf(handler http.HandlerFunc, method string, // a.HTTPBodyNotContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - HTTPBodyNotContains(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyNotContains(a.t, handler, method, url, values, str, msgAndArgs...) } // HTTPBodyNotContainsf asserts that a specified handler returns a @@ -250,8 +349,11 @@ func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string // a.HTTPBodyNotContainsf(myHandler, "www.google.com", nil, "I'm Feeling Lucky", "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { - HTTPBodyNotContainsf(a.t, handler, method, url, values, str) +func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPBodyNotContainsf(a.t, handler, method, url, values, str, msg, args...) } // HTTPError asserts that a specified handler returns an error status code. @@ -259,8 +361,11 @@ func (a *Assertions) HTTPBodyNotContainsf(handler http.HandlerFunc, method strin // a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPError(a.t, handler, method, url, values) +func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPError(a.t, handler, method, url, values, msgAndArgs...) } // HTTPErrorf asserts that a specified handler returns an error status code. @@ -268,8 +373,11 @@ func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url stri // a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPErrorf(a.t, handler, method, url, values) +func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPErrorf(a.t, handler, method, url, values, msg, args...) } // HTTPRedirect asserts that a specified handler returns a redirect status code. @@ -277,8 +385,11 @@ func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url str // a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPRedirect(a.t, handler, method, url, values) +func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPRedirect(a.t, handler, method, url, values, msgAndArgs...) } // HTTPRedirectf asserts that a specified handler returns a redirect status code. @@ -286,8 +397,11 @@ func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url s // a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // // Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). -func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPRedirectf(a.t, handler, method, url, values) +func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPRedirectf(a.t, handler, method, url, values, msg, args...) } // HTTPSuccess asserts that a specified handler returns a success status code. @@ -295,8 +409,11 @@ func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url // a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPSuccess(a.t, handler, method, url, values) +func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPSuccess(a.t, handler, method, url, values, msgAndArgs...) } // HTTPSuccessf asserts that a specified handler returns a success status code. @@ -304,14 +421,20 @@ func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url st // a.HTTPSuccessf(myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") // // Returns whether the assertion was successful (true) or not (false). -func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values) { - HTTPSuccessf(a.t, handler, method, url, values) +func (a *Assertions) HTTPSuccessf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPSuccessf(a.t, handler, method, url, values, msg, args...) } // Implements asserts that an object is implemented by the specified interface. // // a.Implements((*MyInterface)(nil), new(MyObject)) func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Implements(a.t, interfaceObject, object, msgAndArgs...) } @@ -319,86 +442,129 @@ func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, // // a.Implementsf((*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Implementsf(a.t, interfaceObject, object, msg, args...) } // InDelta asserts that the two numerals are within delta of each other. // // a.InDelta(math.Pi, (22 / 7.0), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InDelta(a.t, expected, actual, delta, msgAndArgs...) } +// InDeltaMapValues is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValues(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaMapValues(a.t, expected, actual, delta, msgAndArgs...) +} + +// InDeltaMapValuesf is the same as InDelta, but it compares all values between two maps. Both maps must have exactly the same keys. +func (a *Assertions) InDeltaMapValuesf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + InDeltaMapValuesf(a.t, expected, actual, delta, msg, args...) +} + // InDeltaSlice is the same as InDelta, except it compares two slices. func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) } // InDeltaSlicef is the same as InDelta, except it compares two slices. func (a *Assertions) InDeltaSlicef(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InDeltaSlicef(a.t, expected, actual, delta, msg, args...) } // InDeltaf asserts that the two numerals are within delta of each other. // // a.InDeltaf(math.Pi, (22 / 7.0, "error message %s", "formatted"), 0.01) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InDeltaf(expected interface{}, actual interface{}, delta float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InDeltaf(a.t, expected, actual, delta, msg, args...) } // InEpsilon asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) } // InEpsilonSlice is the same as InEpsilon, except it compares each value from two slices. func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InEpsilonSlice(a.t, expected, actual, epsilon, msgAndArgs...) } // InEpsilonSlicef is the same as InEpsilon, except it compares each value from two slices. func (a *Assertions) InEpsilonSlicef(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InEpsilonSlicef(a.t, expected, actual, epsilon, msg, args...) } // InEpsilonf asserts that expected and actual have a relative error less than epsilon -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) InEpsilonf(expected interface{}, actual interface{}, epsilon float64, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } InEpsilonf(a.t, expected, actual, epsilon, msg, args...) } // IsType asserts that the specified objects are of the same type. func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } IsType(a.t, expectedType, object, msgAndArgs...) } // IsTypef asserts that the specified objects are of the same type. func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } IsTypef(a.t, expectedType, object, msg, args...) } // JSONEq asserts that two JSON strings are equivalent. // // a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } JSONEq(a.t, expected, actual, msgAndArgs...) } // JSONEqf asserts that two JSON strings are equivalent. // // a.JSONEqf(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) JSONEqf(expected string, actual string, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } JSONEqf(a.t, expected, actual, msg, args...) } @@ -406,9 +572,10 @@ func (a *Assertions) JSONEqf(expected string, actual string, msg string, args .. // Len also fails if the object has a type that len() not accept. // // a.Len(mySlice, 3) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Len(a.t, object, length, msgAndArgs...) } @@ -416,27 +583,30 @@ func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface // Lenf also fails if the object has a type that len() not accept. // // a.Lenf(mySlice, 3, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Lenf(object interface{}, length int, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Lenf(a.t, object, length, msg, args...) } // Nil asserts that the specified object is nil. // // a.Nil(err) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Nil(a.t, object, msgAndArgs...) } // Nilf asserts that the specified object is nil. // // a.Nilf(err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Nilf(a.t, object, msg, args...) } @@ -446,9 +616,10 @@ func (a *Assertions) Nilf(object interface{}, msg string, args ...interface{}) { // if a.NoError(err) { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NoError(a.t, err, msgAndArgs...) } @@ -458,9 +629,10 @@ func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) { // if a.NoErrorf(err, "error message %s", "formatted") { // assert.Equal(t, expectedObj, actualObj) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NoErrorf(a.t, err, msg, args...) } @@ -470,9 +642,10 @@ func (a *Assertions) NoErrorf(err error, msg string, args ...interface{}) { // a.NotContains("Hello World", "Earth") // a.NotContains(["Hello", "World"], "Earth") // a.NotContains({"Hello": "World"}, "Earth") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotContains(a.t, s, contains, msgAndArgs...) } @@ -482,9 +655,10 @@ func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs // a.NotContainsf("Hello World", "Earth", "error message %s", "formatted") // a.NotContainsf(["Hello", "World"], "Earth", "error message %s", "formatted") // a.NotContainsf({"Hello": "World"}, "Earth", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotContainsf(a.t, s, contains, msg, args...) } @@ -494,9 +668,10 @@ func (a *Assertions) NotContainsf(s interface{}, contains interface{}, msg strin // if a.NotEmpty(obj) { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotEmpty(a.t, object, msgAndArgs...) } @@ -506,9 +681,10 @@ func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) { // if a.NotEmptyf(obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) // } -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotEmptyf(a.t, object, msg, args...) } @@ -516,11 +692,12 @@ func (a *Assertions) NotEmptyf(object interface{}, msg string, args ...interface // // a.NotEqual(obj1, obj2) // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotEqual(a.t, expected, actual, msgAndArgs...) } @@ -528,47 +705,52 @@ func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndAr // // a.NotEqualf(obj1, obj2, "error message %s", "formatted") // -// Returns whether the assertion was successful (true) or not (false). -// // Pointer variable equality is determined based on the equality of the // referenced values (as opposed to the memory addresses). func (a *Assertions) NotEqualf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotEqualf(a.t, expected, actual, msg, args...) } // NotNil asserts that the specified object is not nil. // // a.NotNil(err) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotNil(a.t, object, msgAndArgs...) } // NotNilf asserts that the specified object is not nil. // // a.NotNilf(err, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotNilf(object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotNilf(a.t, object, msg, args...) } // NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. // // a.NotPanics(func(){ RemainCalm() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotPanics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotPanics(a.t, f, msgAndArgs...) } // NotPanicsf asserts that the code inside the specified PanicTestFunc does NOT panic. // // a.NotPanicsf(func(){ RemainCalm() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotPanicsf(f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotPanicsf(a.t, f, msg, args...) } @@ -576,9 +758,10 @@ func (a *Assertions) NotPanicsf(f assert.PanicTestFunc, msg string, args ...inte // // a.NotRegexp(regexp.MustCompile("starts"), "it's starting") // a.NotRegexp("^start", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotRegexp(a.t, rx, str, msgAndArgs...) } @@ -586,9 +769,10 @@ func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...in // // a.NotRegexpf(regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") // a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotRegexpf(a.t, rx, str, msg, args...) } @@ -596,9 +780,10 @@ func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, arg // elements given in the specified subset(array, slice...). // // a.NotSubset([1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotSubset(a.t, list, subset, msgAndArgs...) } @@ -606,28 +791,36 @@ func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs // elements given in the specified subset(array, slice...). // // a.NotSubsetf([1, 3, 4], [1, 2], "But [1, 3, 4] does not contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotSubsetf(a.t, list, subset, msg, args...) } -// NotZero asserts that i is not the zero value for its type and returns the truth. +// NotZero asserts that i is not the zero value for its type. func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotZero(a.t, i, msgAndArgs...) } -// NotZerof asserts that i is not the zero value for its type and returns the truth. +// NotZerof asserts that i is not the zero value for its type. func (a *Assertions) NotZerof(i interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } NotZerof(a.t, i, msg, args...) } // Panics asserts that the code inside the specified PanicTestFunc panics. // // a.Panics(func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Panics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Panics(a.t, f, msgAndArgs...) } @@ -635,9 +828,10 @@ func (a *Assertions) Panics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { // the recovered panic value equals the expected panic value. // // a.PanicsWithValue("crazy error", func(){ GoCrazy() }) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) PanicsWithValue(expected interface{}, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } PanicsWithValue(a.t, expected, f, msgAndArgs...) } @@ -645,18 +839,20 @@ func (a *Assertions) PanicsWithValue(expected interface{}, f assert.PanicTestFun // the recovered panic value equals the expected panic value. // // a.PanicsWithValuef("crazy error", func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) PanicsWithValuef(expected interface{}, f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } PanicsWithValuef(a.t, expected, f, msg, args...) } // Panicsf asserts that the code inside the specified PanicTestFunc panics. // // a.Panicsf(func(){ GoCrazy() }, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Panicsf(f assert.PanicTestFunc, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Panicsf(a.t, f, msg, args...) } @@ -664,9 +860,10 @@ func (a *Assertions) Panicsf(f assert.PanicTestFunc, msg string, args ...interfa // // a.Regexp(regexp.MustCompile("start"), "it's starting") // a.Regexp("start...$", "it's not starting") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Regexp(a.t, rx, str, msgAndArgs...) } @@ -674,9 +871,10 @@ func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...inter // // a.Regexpf(regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") // a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Regexpf(a.t, rx, str, msg, args...) } @@ -684,9 +882,10 @@ func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args . // elements given in the specified subset(array, slice...). // // a.Subset([1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Subset(a.t, list, subset, msgAndArgs...) } @@ -694,54 +893,65 @@ func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ... // elements given in the specified subset(array, slice...). // // a.Subsetf([1, 2, 3], [1, 2], "But [1, 2, 3] does contain [1, 2]", "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Subsetf(a.t, list, subset, msg, args...) } // True asserts that the specified value is true. // // a.True(myBool) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) True(value bool, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } True(a.t, value, msgAndArgs...) } // Truef asserts that the specified value is true. // // a.Truef(myBool, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) Truef(value bool, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Truef(a.t, value, msg, args...) } // WithinDuration asserts that the two times are within duration delta of each other. // // a.WithinDuration(time.Now(), time.Now(), 10*time.Second) -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } WithinDuration(a.t, expected, actual, delta, msgAndArgs...) } // WithinDurationf asserts that the two times are within duration delta of each other. // // a.WithinDurationf(time.Now(), time.Now(), 10*time.Second, "error message %s", "formatted") -// -// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) WithinDurationf(expected time.Time, actual time.Time, delta time.Duration, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } WithinDurationf(a.t, expected, actual, delta, msg, args...) } -// Zero asserts that i is the zero value for its type and returns the truth. +// Zero asserts that i is the zero value for its type. func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Zero(a.t, i, msgAndArgs...) } -// Zerof asserts that i is the zero value for its type and returns the truth. +// Zerof asserts that i is the zero value for its type. func (a *Assertions) Zerof(i interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } Zerof(a.t, i, msg, args...) } diff --git a/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl b/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl index b93569e0a971..54124df1d3bb 100644 --- a/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl +++ b/vendor/github.com/stretchr/testify/require/require_forward.go.tmpl @@ -1,4 +1,5 @@ {{.CommentWithoutT "a"}} func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) { + if h, ok := a.t.(tHelper); ok { h.Helper() } {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) } diff --git a/vendor/github.com/stretchr/testify/require/requirements.go b/vendor/github.com/stretchr/testify/require/requirements.go index e404f016d182..690583a8e03e 100644 --- a/vendor/github.com/stretchr/testify/require/requirements.go +++ b/vendor/github.com/stretchr/testify/require/requirements.go @@ -6,4 +6,24 @@ type TestingT interface { FailNow() } +type tHelper interface { + Helper() +} + +// ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful +// for table driven tests. +type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) + +// ValueAssertionFunc is a common function prototype when validating a single value. Can be useful +// for table driven tests. +type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) + +// BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful +// for table driven tests. +type BoolAssertionFunc func(TestingT, bool, ...interface{}) + +// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// for table driven tests. +type ErrorAssertionFunc func(TestingT, error, ...interface{}) + //go:generate go run ../_codegen/main.go -output-package=require -template=require.go.tmpl -include-format-funcs diff --git a/vendor/vendor.json b/vendor/vendor.json index 79014dbe4347..9769cfa3d93a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -288,9 +288,9 @@ {"path":"github.com/skratchdot/open-golang/open","checksumSHA1":"h/HMhokbQHTdLUbruoBBTee+NYw=","revision":"75fb7ed4208cf72d323d7d02fd1a5964a7a9073c","revisionTime":"2016-03-02T14:40:31Z"}, {"path":"github.com/spf13/pflag","checksumSHA1":"Q52Y7t0lEtk/wcDn5q7tS7B+jqs=","revision":"7aff26db30c1be810f9de5038ec5ef96ac41fd7c","revisionTime":"2017-08-24T17:57:12Z"}, {"path":"github.com/stretchr/objx","checksumSHA1":"K0crHygPTP42i1nLKWphSlvOQJw=","revision":"1a9d0bb9f541897e62256577b352fdbc1fb4fd94","revisionTime":"2015-09-28T12:21:52Z"}, - {"path":"github.com/stretchr/testify/assert","checksumSHA1":"5NBHAe3S15q3L9hOLThnMZjIZRE=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"}, - {"path":"github.com/stretchr/testify/mock","checksumSHA1":"o+jsS/rxceTym4M3reSPfrPxaio=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"}, - {"path":"github.com/stretchr/testify/require","checksumSHA1":"7vs6dSc1PPGBKyzb/SCIyeMJPLQ=","revision":"f6abca593680b2315d2075e0f5e2a9751e3f431a","revisionTime":"2017-06-01T20:57:54Z"}, + {"path":"github.com/stretchr/testify/assert","checksumSHA1":"6LwXZI7kXm1C0h4Ui0Y52p9uQhk=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"}, + {"path":"github.com/stretchr/testify/mock","checksumSHA1":"Qloi2PTvZv+D9FDHXM/banCoaFY=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"}, + {"path":"github.com/stretchr/testify/require","checksumSHA1":"KqYmXUcuGwsvBL6XVsQnXsFb3LI=","revision":"c679ae2cc0cb27ec3293fea7e254e47386f05d69","revisionTime":"2018-03-14T08:05:35Z"}, {"path":"github.com/syndtr/gocapability/capability","checksumSHA1":"PgEklGW56c5RLHqQhORxt6jS3fY=","revision":"db04d3cc01c8b54962a58ec7e491717d06cfcc16","revisionTime":"2017-07-04T07:02:18Z"}, {"path":"github.com/tonnerre/golang-text","checksumSHA1":"t24KnvC9jRxiANVhpw2pqFpmEu8=","revision":"048ed3d792f7104850acbc8cfc01e5a6070f4c04","revisionTime":"2013-09-25T19:58:46Z"}, {"path":"github.com/ugorji/go/codec","checksumSHA1":"8G1zvpE4gTtWQRuP/x2HPVDmflo=","revision":"0053ebfd9d0ee06ccefbfe17072021e1d4acebee","revisionTime":"2017-06-20T06:01:02Z"}, diff --git a/website/source/api/json-jobs.html.md b/website/source/api/json-jobs.html.md index ef86e1551460..27bb00756a44 100644 --- a/website/source/api/json-jobs.html.md +++ b/website/source/api/json-jobs.html.md @@ -620,12 +620,19 @@ determined. The potential values are: - `MinHealthyTime` - Specifies the minimum time the allocation must be in the healthy state before it is marked as healthy and unblocks further allocations - from being updated. This is specified using a label suffix like "30s" or - "15m". + from being updated. - `HealthyDeadline` - Specifies the deadline in which the allocation must be marked as healthy after which the allocation is automatically transitioned to - unhealthy. This is specified using a label suffix like "2m" or "1h". + unhealthy. + +- `ProgressDeadline` - Specifies the deadline in which an allocation must be + marked as healthy. The deadline begins when the first allocation for the + deployment is created and is reset whenever an allocation as part of the + deployment transitions to a healthy state. If no allocation transitions to the + healthy state before the progress deadline, the deployment is marked as + failed. If the `progress_deadline` is set to `0`, the first allocation to be + marked as unhealthy causes the deployment to fail. - `AutoRevert` - Specifies if the job should auto-revert to the last stable job on deployment failure. A job is marked as stable if all the allocations as @@ -638,7 +645,7 @@ determined. The potential values are: allocations at a rate of `max_parallel`. - `Stagger` - Specifies the delay between migrating allocations off nodes marked - for draining. This is specified using a label suffix like "30s" or "1h". + for draining. An example `Update` block: diff --git a/website/source/docs/job-specification/update.html.md b/website/source/docs/job-specification/update.html.md index 73811f6cf297..c774ce7f5cc4 100644 --- a/website/source/docs/job-specification/update.html.md +++ b/website/source/docs/job-specification/update.html.md @@ -31,13 +31,14 @@ highest precedence and then the job. ```hcl job "docs" { update { - max_parallel = 3 - health_check = "checks" - min_healthy_time = "10s" - healthy_deadline = "10m" - auto_revert = true - canary = 1 - stagger = "30s" + max_parallel = 3 + health_check = "checks" + min_healthy_time = "10s" + healthy_deadline = "5m" + progress_deadline = "10m" + auto_revert = true + canary = 1 + stagger = "30s" } } ``` @@ -77,6 +78,15 @@ set of updates. The `system` scheduler will be updated to support the new automatically transitioned to unhealthy. This is specified using a label suffix like "2m" or "1h". +- `progress_deadline` `(string: "10m")` - Specifies the deadline in which an + allocation must be marked as healthy. The deadline begins when the first + allocation for the deployment is created and is reset whenever an allocation + as part of the deployment transitions to a healthy state. If no allocation + transitions to the healthy state before the progress deadline, the deployment + is marked as failed. If the `progress_deadline` is set to `0`, the first + allocation to be marked as unhealthy causes the deployment to fail. This is + specified using a label suffix like "2m" or "1h". + - `auto_revert` `(bool: false)` - Specifies if the job should auto-revert to the last stable job on deployment failure. A job is marked as stable if all the allocations as part of its deployment were marked healthy.