-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Makes auto reverts robust against previously stable job that fail to start correctly upon reverting #3496
Merged
Merged
Makes auto reverts robust against previously stable job that fail to start correctly upon reverting #3496
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
f83c540
Fixes auto revert to check if the job's spec has changed before rever…
preetapan 99c57ee
Update rollback test to add a spec change, and add new test for rollb…
preetapan 48844e0
Remove extra newline
preetapan ba6c808
Clarify comment about infinite revert cycles
preetapan 145a959
Adds SpecChanged check to alloc health and fail deployment end points…
preetapan 05af7bd
Check that job version doesn't change when rollback does not occur du…
preetapan f92e564
Added more unit tests for testing rollback when job has identical spe…
preetapan 04feeef
Adds comment to handleRollbackValidity method and other small test re…
preetapan File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -281,6 +281,9 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { | |
// Upsert the job again to get a new version | ||
j2 := j.Copy() | ||
j2.Stable = false | ||
// Modify the job to make its specification different | ||
j2.Meta["foo"] = "bar" | ||
|
||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") | ||
|
||
w.SetEnabled(true, m.state) | ||
|
@@ -316,6 +319,66 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { | |
m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1) | ||
} | ||
|
||
// Test setting allocation unhealthy on job with identical spec and there should be no rollback | ||
func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) { | ||
t.Parallel() | ||
assert := assert.New(t) | ||
w, m := defaultTestDeploymentWatcher(t) | ||
|
||
// Create a job, alloc, and a deployment | ||
j := mock.Job() | ||
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() | ||
j.TaskGroups[0].Update.MaxParallel = 2 | ||
j.TaskGroups[0].Update.AutoRevert = true | ||
j.Stable = true | ||
d := mock.Deployment() | ||
d.JobID = j.ID | ||
d.TaskGroups["web"].AutoRevert = true | ||
a := mock.Alloc() | ||
a.DeploymentID = d.ID | ||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") | ||
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") | ||
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") | ||
|
||
// Upsert the job again to get a new version | ||
j2 := j.Copy() | ||
j2.Stable = false | ||
|
||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") | ||
|
||
w.SetEnabled(true, m.state) | ||
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, | ||
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) | ||
|
||
// Assert that we get a call to UpsertDeploymentAllocHealth | ||
matchConfig := &matchDeploymentAllocHealthRequestConfig{ | ||
DeploymentID: d.ID, | ||
Unhealthy: []string{a.ID}, | ||
Eval: true, | ||
DeploymentUpdate: &structs.DeploymentStatusUpdate{ | ||
DeploymentID: d.ID, | ||
Status: structs.DeploymentStatusFailed, | ||
StatusDescription: structs.DeploymentStatusDescriptionFailedAllocations, | ||
}, | ||
JobVersion: nil, | ||
} | ||
matcher := matchDeploymentAllocHealthRequest(matchConfig) | ||
m.On("UpdateDeploymentAllocHealth", mocker.MatchedBy(matcher)).Return(nil) | ||
|
||
// Call SetAllocHealth | ||
req := &structs.DeploymentAllocHealthRequest{ | ||
DeploymentID: d.ID, | ||
UnhealthyAllocationIDs: []string{a.ID}, | ||
} | ||
var resp structs.DeploymentUpdateResponse | ||
err := w.SetAllocHealth(req, &resp) | ||
assert.Nil(err, "SetAllocHealth") | ||
|
||
testutil.WaitForResult(func() (bool, error) { return 0 == len(w.watchers), nil }, | ||
func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) | ||
m.AssertNumberOfCalls(t, "UpdateDeploymentAllocHealth", 1) | ||
} | ||
|
||
// Test promoting a deployment | ||
func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { | ||
t.Parallel() | ||
|
@@ -639,6 +702,8 @@ func TestDeploymentWatcher_Watch(t *testing.T) { | |
|
||
// Upsert the job again to get a new version | ||
j2 := j.Copy() | ||
// Modify the job to make its specification different | ||
j2.Meta["foo"] = "bar" | ||
j2.Stable = false | ||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") | ||
|
||
|
@@ -734,6 +799,117 @@ func TestDeploymentWatcher_Watch(t *testing.T) { | |
func(err error) { assert.Equal(0, len(w.watchers), "Should have no deployment") }) | ||
} | ||
|
||
// Tests that the watcher fails rollback when the spec hasn't changed | ||
func TestDeploymentWatcher_RollbackFailed(t *testing.T) { | ||
t.Parallel() | ||
assert := assert.New(t) | ||
w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) | ||
|
||
// Create a job, alloc, and a deployment | ||
j := mock.Job() | ||
j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() | ||
j.TaskGroups[0].Update.MaxParallel = 2 | ||
j.TaskGroups[0].Update.AutoRevert = true | ||
j.Stable = true | ||
d := mock.Deployment() | ||
d.JobID = j.ID | ||
d.TaskGroups["web"].AutoRevert = true | ||
a := mock.Alloc() | ||
a.DeploymentID = d.ID | ||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j), "UpsertJob") | ||
assert.Nil(m.state.UpsertDeployment(m.nextIndex(), d), "UpsertDeployment") | ||
assert.Nil(m.state.UpsertAllocs(m.nextIndex(), []*structs.Allocation{a}), "UpsertAllocs") | ||
|
||
// Upsert the job again to get a new version | ||
j2 := j.Copy() | ||
// Modify the job to make its specification different | ||
j2.Stable = false | ||
assert.Nil(m.state.UpsertJob(m.nextIndex(), j2), "UpsertJob2") | ||
|
||
w.SetEnabled(true, m.state) | ||
testutil.WaitForResult(func() (bool, error) { return 1 == len(w.watchers), nil }, | ||
func(err error) { assert.Equal(1, len(w.watchers), "Should have 1 deployment") }) | ||
|
||
// Assert that we will get a createEvaluation call only once. This will | ||
// verify that the watcher is batching allocation changes | ||
m1 := matchUpsertEvals([]string{d.ID}) | ||
m.On("UpsertEvals", mocker.MatchedBy(m1)).Return(nil).Once() | ||
|
||
// Update the allocs health to healthy which should create an evaluation | ||
for i := 0; i < 5; i++ { | ||
req := &structs.ApplyDeploymentAllocHealthRequest{ | ||
DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ | ||
DeploymentID: d.ID, | ||
HealthyAllocationIDs: []string{a.ID}, | ||
}, | ||
} | ||
assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req), "UpsertDeploymentAllocHealth") | ||
} | ||
|
||
// Wait for there to be one eval | ||
testutil.WaitForResult(func() (bool, error) { | ||
ws := memdb.NewWatchSet() | ||
evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID) | ||
if err != nil { | ||
return false, err | ||
} | ||
|
||
if l := len(evals); l != 1 { | ||
return false, fmt.Errorf("Got %d evals; want 1", l) | ||
} | ||
|
||
return true, nil | ||
}, func(err error) { | ||
t.Fatal(err) | ||
}) | ||
|
||
// Assert that we get a call to UpsertDeploymentStatusUpdate with roll back failed as the status | ||
c := &matchDeploymentStatusUpdateConfig{ | ||
DeploymentID: d.ID, | ||
Status: structs.DeploymentStatusFailed, | ||
StatusDescription: structs.DeploymentStatusDescriptionRollbackNoop(structs.DeploymentStatusDescriptionFailedAllocations, 0), | ||
JobVersion: nil, | ||
Eval: true, | ||
} | ||
m2 := matchDeploymentStatusUpdateRequest(c) | ||
m.On("UpdateDeploymentStatus", mocker.MatchedBy(m2)).Return(nil) | ||
|
||
// Update the allocs health to unhealthy which will cause attempting a rollback, | ||
// fail in that step, do status update and eval | ||
req2 := &structs.ApplyDeploymentAllocHealthRequest{ | ||
DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ | ||
DeploymentID: d.ID, | ||
UnhealthyAllocationIDs: []string{a.ID}, | ||
}, | ||
} | ||
assert.Nil(m.state.UpdateDeploymentAllocHealth(m.nextIndex(), req2), "UpsertDeploymentAllocHealth") | ||
|
||
// Wait for there to be one eval | ||
testutil.WaitForResult(func() (bool, error) { | ||
ws := memdb.NewWatchSet() | ||
evals, err := m.state.EvalsByJob(ws, j.Namespace, j.ID) | ||
if err != nil { | ||
return false, err | ||
} | ||
|
||
if l := len(evals); l != 2 { | ||
return false, fmt.Errorf("Got %d evals; want 1", l) | ||
} | ||
|
||
return true, nil | ||
}, func(err error) { | ||
t.Fatal(err) | ||
}) | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Don't we need to assert the job version hasn't changed? |
||
m.AssertCalled(t, "UpsertEvals", mocker.MatchedBy(m1)) | ||
|
||
// verify that the job version hasn't changed after upsert | ||
m.state.JobByID(nil, structs.DefaultNamespace, j.ID) | ||
if j.Version != 0 { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use assert where possible |
||
t.Fatalf("Expected job version 0 but got %v", j.Version) | ||
} | ||
} | ||
|
||
// Test evaluations are batched between watchers | ||
func TestWatcher_BatchEvals(t *testing.T) { | ||
t.Parallel() | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Comment on the inputs and outputs