Skip to content

Commit

Permalink
scheduler: sort gang of same gangGroup by gangId
Browse files Browse the repository at this point in the history
Signed-off-by: wangjianyu.wjy <wangjianyu.wjy@alibaba-inc.com>
  • Loading branch information
wangjianyu.wjy committed Apr 11, 2024
1 parent 10567b8 commit 1a12fe6
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 22 deletions.
9 changes: 3 additions & 6 deletions pkg/scheduler/plugins/coscheduling/core/gang.go
Original file line number Diff line number Diff line change
Expand Up @@ -390,12 +390,9 @@ func (gang *Gang) setScheduleCycleValid(valid bool) {

if !valid && !gang.ScheduleCycleValid {
/*
let's explain why by following example, there are three gang of one group: A, B and C, every gang group have min-member of 10 pod, noted as A1-A10, B1-B10, C1-C10.
1. A1-A5 assumed in gangA scheduleCycle 1, B1-B5 assumed in gangA scheduleCycle 1, C1-C5 assumed in gangA scheduleCycle 1, C6 filter failed due to insufficient resource, then A6-10 failed due to cycle invalid, C7-C10 failed due to cycle invalid, B6-B9 failed due to cycle invalid
2. A1-A5 assumed in gangA scheduleCycle 2, C1-C5 assumed in gangA scheduleCycle 2, C6 failed due to insufficient resource and gangA\B\C cycle set to invalid,
3. then A6-A10,C7-C10 failed due to cycle invalid
4. then B10 failed due to cycle invalid, B1 comes and find its gang scheduling cycle 2 can be valid, so it's assumed.
5. however all it's sibling will come until next round of all gangs, these gangs will face insufficient resource due to pre-allocated of B10
let's explain why by following example, there are three gang of one group: F, G, every gang group have min-member of 10 pod, noted as F1-F10, G1-G10.
1. F1 failed due to insufficient resource, F2-F10 failed due to cycle invalid,
2. then G1-G10 assumed, however all it's sibling will come until next round of all gangs, these gangs will face insufficient resource due to pre-allocated of G1-G10
TODO this logic can be optimized by give gangs of same group same scheduling cycle
*/
Expand Down
21 changes: 8 additions & 13 deletions pkg/scheduler/plugins/coscheduling/coscheduling.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,20 +140,15 @@ func (cs *Coscheduling) Less(podInfo1, podInfo2 *framework.QueuedPodInfo) bool {
}
gangId1 := util.GetId(podInfo1.Pod.Namespace, util.GetGangNameByPod(podInfo1.Pod))
gangId2 := util.GetId(podInfo2.Pod.Namespace, util.GetGangNameByPod(podInfo2.Pod))
// for member gang of same gangGroup, the gang that haven’t been satisfied yet take precedence

if gangId1 != gangId2 {
isGang1Satisfied := cs.pgMgr.IsGangMinSatisfied(podInfo1.Pod)
isGang2Satisfied := cs.pgMgr.IsGangMinSatisfied(podInfo2.Pod)
if isGang1Satisfied != isGang2Satisfied {
return !isGang1Satisfied
}
} else {
// for member pod of same gang, the pod with the smaller scheduling cycle take precedence so that gang scheduling cycle can be valid and iterated
childScheduleCycle1 := cs.pgMgr.GetChildScheduleCycle(podInfo1.Pod)
childScheduleCycle2 := cs.pgMgr.GetChildScheduleCycle(podInfo2.Pod)
if childScheduleCycle1 != childScheduleCycle2 {
return childScheduleCycle1 < childScheduleCycle2
}
return gangId1 < gangId2
}
// for member pod of same gang, the pod with the smaller scheduling cycle take precedence so that gang scheduling cycle can be valid and iterated
childScheduleCycle1 := cs.pgMgr.GetChildScheduleCycle(podInfo1.Pod)
childScheduleCycle2 := cs.pgMgr.GetChildScheduleCycle(podInfo2.Pod)
if childScheduleCycle1 != childScheduleCycle2 {
return childScheduleCycle1 < childScheduleCycle2
}
return podInfo1.Timestamp.Before(podInfo2.Timestamp)
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/scheduler/plugins/coscheduling/coscheduling_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,7 @@ func TestFairness(t *testing.T) {
"gangJ": {"default/gangH", "default/gangI", "default/gangJ"},
}
gangMinRequiredNums := []int{20, 10, 32, 20, 20, 18, 43, 20, 30, 20}
gangInjectFilterError := []bool{false, false, true, false, false, true, true, false, false, true}
gangInjectFilterError := []bool{false, false, true, false, false, true, false, false, false, true}
var gangInjectFilterErrorIndex []int
for _, gangMinRequiredNum := range gangMinRequiredNums {
gangInjectFilterErrorIndex = append(gangInjectFilterErrorIndex, rand.Intn(gangMinRequiredNum))
Expand Down Expand Up @@ -996,8 +996,8 @@ func TestFairness(t *testing.T) {
ctx.Done(),
scheduler.WithProfiles(cfg.Profiles...),
scheduler.WithFrameworkOutOfTreeRegistry(registry),
scheduler.WithPodInitialBackoffSeconds(1),
scheduler.WithPodMaxBackoffSeconds(1),
scheduler.WithPodInitialBackoffSeconds(0),
scheduler.WithPodMaxBackoffSeconds(0),
scheduler.WithPodMaxInUnschedulablePodsDuration(0),
)
assert.NoError(t, err)
Expand Down

0 comments on commit 1a12fe6

Please sign in to comment.