-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Client only pulls update allocations from server #731
Changes from 2 commits
410ae59
3b90539
c0ab75a
679a57c
019883c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,7 +38,8 @@ type AllocRunner struct { | |
logger *log.Logger | ||
consulService *ConsulService | ||
|
||
alloc *structs.Allocation | ||
alloc *structs.Allocation | ||
allocLock sync.Mutex | ||
|
||
dirtyCh chan struct{} | ||
|
||
|
@@ -155,6 +156,8 @@ func (r *AllocRunner) SaveState() error { | |
func (r *AllocRunner) saveAllocRunnerState() error { | ||
r.taskStatusLock.RLock() | ||
defer r.taskStatusLock.RUnlock() | ||
r.allocLock.Lock() | ||
defer r.allocLock.Unlock() | ||
snap := allocRunnerState{ | ||
Alloc: r.alloc, | ||
RestartPolicy: r.RestartPolicy, | ||
|
@@ -184,6 +187,8 @@ func (r *AllocRunner) DestroyContext() error { | |
|
||
// Alloc returns the associated allocation | ||
func (r *AllocRunner) Alloc() *structs.Allocation { | ||
r.allocLock.Lock() | ||
defer r.allocLock.Unlock() | ||
return r.alloc | ||
} | ||
|
||
|
@@ -221,7 +226,8 @@ func (r *AllocRunner) syncStatus() error { | |
// Scan the task states to determine the status of the alloc | ||
var pending, running, dead, failed bool | ||
r.taskStatusLock.RLock() | ||
for _, state := range r.alloc.TaskStates { | ||
for _, tr := range r.tasks { | ||
state := tr.state | ||
switch state.State { | ||
case structs.TaskStateRunning: | ||
running = true | ||
|
@@ -236,13 +242,17 @@ func (r *AllocRunner) syncStatus() error { | |
} | ||
} | ||
} | ||
r.taskStatusLock.RUnlock() | ||
|
||
// Determine the alloc status | ||
r.allocLock.Lock() | ||
defer r.allocLock.Unlock() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Soon after this, we are calling the updater method to update the alloc and that makes an RPC call. Have we considered the implications of locking the alloc during the time the rpc happens? On a bad day/network it might be several seconds which means a lot of backed up operations on the client. |
||
|
||
if len(r.alloc.TaskStates) > 0 { | ||
taskDesc, _ := json.Marshal(r.alloc.TaskStates) | ||
r.alloc.ClientDescription = string(taskDesc) | ||
} | ||
r.taskStatusLock.RUnlock() | ||
|
||
// Determine the alloc status | ||
if failed { | ||
r.alloc.ClientStatus = structs.AllocClientStatusFailed | ||
} else if running { | ||
|
@@ -336,6 +346,11 @@ OUTER: | |
for { | ||
select { | ||
case update := <-r.updateCh: | ||
// Store the updated allocation. | ||
r.allocLock.Lock() | ||
r.alloc = update | ||
r.allocLock.Unlock() | ||
|
||
// Check if we're in a terminal status | ||
if update.TerminalStatus() { | ||
break OUTER | ||
|
@@ -371,8 +386,8 @@ OUTER: | |
} | ||
|
||
// Destroy each sub-task | ||
r.taskLock.RLock() | ||
defer r.taskLock.RUnlock() | ||
r.taskLock.Lock() | ||
defer r.taskLock.Unlock() | ||
for _, tr := range r.tasks { | ||
tr.Destroy() | ||
} | ||
|
@@ -408,6 +423,14 @@ func (r *AllocRunner) Update(update *structs.Allocation) { | |
} | ||
} | ||
|
||
// shouldUpdate takes the AllocModifyIndex of an allocation sent from the server and | ||
// checks if the current running allocation is behind and should be updated. | ||
func (r *AllocRunner) shouldUpdate(serverIndex uint64) bool { | ||
r.allocLock.Lock() | ||
defer r.allocLock.Unlock() | ||
return r.alloc.AllocModifyIndex < serverIndex | ||
} | ||
|
||
// Destroy is used to indicate that the allocation context should be destroyed | ||
func (r *AllocRunner) Destroy() { | ||
r.destroyLock.Lock() | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why do we need to acquire a lock here? I don't see us mutating anything here. Does the caller expects the alloc to not change once it has? We should probably make a copy and return if that's the assumption, otherwise the alloc might change under the hood once the caller has it.