Skip to content

Commit

Permalink
scheduler: add more quota details used and request in quotaInfo (#2001)
Browse files Browse the repository at this point in the history 
Signed-off-by: xulinfei.xlf <xulinfei.xlf@alibaba-inc.com>
Co-authored-by: xulinfei.xlf <xulinfei.xlf@alibaba-inc.com>
  • Loading branch information
@xulinfei1996
xulinfei1996 and xulinfei.xlf committed Apr 15, 2024
1 parent 2fb6fc4 commit e64e74b
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 55 deletions.
29 changes: 18 additions & 11 deletions pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,8 @@ func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq,
for i := 0; i < len(curToAllParInfos); i++ {
curQuotaInfo := curToAllParInfos[i]
oldSubLimitReq := curQuotaInfo.getLimitRequestNoLock()
curQuotaInfo.addRequestNonNegativeNoLock(deltaReq, deltaNonPreemptibleRequest)
isSelfRequest := i == 0
curQuotaInfo.addRequestNonNegativeNoLock(deltaReq, deltaNonPreemptibleRequest, isSelfRequest)
if curQuotaInfo.Name == extension.RootQuotaName {
return
}
Expand Down Expand Up @@ -234,7 +235,8 @@ func (gqm *GroupQuotaManager) updateGroupDeltaUsedNoLock(quotaName string, delta
defer gqm.scopedLockForQuotaInfo(curToAllParInfos)()
for i := 0; i < allQuotaInfoLen; i++ {
quotaInfo := curToAllParInfos[i]
quotaInfo.addUsedNonNegativeNoLock(delta, deltaNonPreemptibleUsed)
isSelfUsed := i == 0
quotaInfo.addUsedNonNegativeNoLock(delta, deltaNonPreemptibleUsed, isSelfUsed)
}

if utilfeature.DefaultFeatureGate.Enabled(features.ElasticQuotaGuaranteeUsage) {
Expand Down Expand Up @@ -338,7 +340,7 @@ func (gqm *GroupQuotaManager) getCurToAllParentGroupQuotaInfoNoLock(quotaName st
return curToAllParInfos
}

for true {
for {
curToAllParInfos = append(curToAllParInfos, quotaInfo)
if quotaInfo.Name == extension.RootQuotaName {
break
Expand Down Expand Up @@ -472,7 +474,7 @@ func (gqm *GroupQuotaManager) buildSubParGroupTopoNoLock() {

// ResetAllGroupQuotaNoLock no need to lock gqm.lock
func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() {
childRequestMap, childNonPreemptibleUsedMap, childUsedMap, childNonPreemptibleRequestMap :=
toUpdateRequestMap, toUpdateNonPreemptibleUsedMap, toUpdateUsedMap, toUpdateNonPreemptibleRequestMap :=
make(quotaResMapType), make(quotaResMapType), make(quotaResMapType), make(quotaResMapType)
for quotaName, topoNode := range gqm.quotaTopoNodeMap {
if quotaName == extension.RootQuotaName {
Expand All @@ -481,10 +483,15 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() {
}
topoNode.quotaInfo.lock.Lock()
if !topoNode.quotaInfo.IsParent {
childRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.ChildRequest.DeepCopy()
childNonPreemptibleRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleRequest.DeepCopy()
childNonPreemptibleUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleUsed.DeepCopy()
childUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.Used.DeepCopy()
toUpdateRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.ChildRequest.DeepCopy()
toUpdateNonPreemptibleRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleRequest.DeepCopy()
toUpdateNonPreemptibleUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.NonPreemptibleUsed.DeepCopy()
toUpdateUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.Used.DeepCopy()
} else {
toUpdateRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.SelfRequest.DeepCopy()
toUpdateNonPreemptibleRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.SelfNonPreemptibleRequest.DeepCopy()
toUpdateNonPreemptibleUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.SelfNonPreemptibleUsed.DeepCopy()
toUpdateUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.SelfUsed.DeepCopy()
}
topoNode.quotaInfo.clearForResetNoLock()
topoNode.quotaInfo.lock.Unlock()
Expand All @@ -501,9 +508,9 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() {

// subGroup's topo relation may change; refresh the request/used from bottom to top
for quotaName, topoNode := range gqm.quotaTopoNodeMap {
if !topoNode.quotaInfo.IsParent {
gqm.updateGroupDeltaRequestNoLock(quotaName, childRequestMap[quotaName], childNonPreemptibleRequestMap[quotaName])
gqm.updateGroupDeltaUsedNoLock(quotaName, childUsedMap[quotaName], childNonPreemptibleUsedMap[quotaName])
if _, ok := toUpdateRequestMap[topoNode.quotaInfo.Name]; ok {
gqm.updateGroupDeltaRequestNoLock(quotaName, toUpdateRequestMap[quotaName], toUpdateNonPreemptibleRequestMap[quotaName])
gqm.updateGroupDeltaUsedNoLock(quotaName, toUpdateUsedMap[quotaName], toUpdateNonPreemptibleUsedMap[quotaName])
}
}
}
Expand Down
88 changes: 88 additions & 0 deletions pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,8 @@ func TestGroupQuotaManager_UpdateGroupDeltaUsedAndNonPreemptibleUsed(t *testing.
assert.NotNil(t, quotaInfo)
assert.Equal(t, used, quotaInfo.CalculateInfo.Used)
assert.Equal(t, nonPreemptibleUsed, quotaInfo.CalculateInfo.NonPreemptibleUsed)
assert.Equal(t, nonPreemptibleUsed, quotaInfo.CalculateInfo.SelfNonPreemptibleUsed)
assert.Equal(t, used, quotaInfo.CalculateInfo.SelfUsed)

// 2. used increases to [130,300]
used = createResourceList(10, 10*GigaByte)
Expand All @@ -452,6 +454,8 @@ func TestGroupQuotaManager_UpdateGroupDeltaUsedAndNonPreemptibleUsed(t *testing.
assert.NotNil(t, quotaInfo)
assert.Equal(t, createResourceList(130, 300*GigaByte), quotaInfo.CalculateInfo.Used)
assert.Equal(t, createResourceList(30, 40*GigaByte), quotaInfo.CalculateInfo.NonPreemptibleUsed)
assert.Equal(t, createResourceList(130, 300*GigaByte), quotaInfo.CalculateInfo.SelfUsed)
assert.Equal(t, createResourceList(30, 40*GigaByte), quotaInfo.CalculateInfo.SelfNonPreemptibleUsed)

// 3. used decreases to [90,100]
used = createResourceList(-40, -200*GigaByte)
Expand All @@ -461,6 +465,8 @@ func TestGroupQuotaManager_UpdateGroupDeltaUsedAndNonPreemptibleUsed(t *testing.
assert.NotNil(t, quotaInfo)
assert.Equal(t, createResourceList(90, 100*GigaByte), quotaInfo.CalculateInfo.Used)
assert.Equal(t, createResourceList(15, 20*GigaByte), quotaInfo.CalculateInfo.NonPreemptibleUsed)
assert.Equal(t, createResourceList(90, 100*GigaByte), quotaInfo.CalculateInfo.SelfUsed)
assert.Equal(t, createResourceList(15, 20*GigaByte), quotaInfo.CalculateInfo.SelfNonPreemptibleUsed)
}

func TestGroupQuotaManager_MultiQuotaAdd(t *testing.T) {
Expand Down Expand Up @@ -797,6 +803,24 @@ func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) {
request := createResourceList(60, 100*GigaByte)
gqm.updateGroupDeltaRequestNoLock("a-123", request, request)
gqm.updateGroupDeltaUsedNoLock("a-123", request, createResourceList(0, 0))
qi1 := gqm.GetQuotaInfoByName("test1")
qi2 := gqm.GetQuotaInfoByName("test1-a")
qi3 := gqm.GetQuotaInfoByName("a-123")
qi4 := gqm.GetQuotaInfoByName("test2")
qi5 := gqm.GetQuotaInfoByName("test2-a")
assert.Equal(t, request, qi1.CalculateInfo.Request)
assert.Equal(t, request, qi1.CalculateInfo.Used)
assert.Equal(t, request, qi2.CalculateInfo.Request)
assert.Equal(t, request, qi2.CalculateInfo.Used)
assert.Equal(t, request, qi3.CalculateInfo.SelfRequest)
assert.Equal(t, request, qi3.CalculateInfo.SelfUsed)
assert.Equal(t, v1.ResourceList{}, qi1.CalculateInfo.SelfRequest)
assert.Equal(t, v1.ResourceList{}, qi1.CalculateInfo.SelfUsed)
assert.Equal(t, v1.ResourceList{}, qi2.CalculateInfo.SelfRequest)
assert.Equal(t, v1.ResourceList{}, qi2.CalculateInfo.SelfUsed)
assert.Equal(t, request, qi3.CalculateInfo.SelfRequest)
assert.Equal(t, request, qi3.CalculateInfo.SelfUsed)

runtime := gqm.RefreshRuntime("a-123")
assert.Equal(t, request, runtime)

Expand All @@ -810,6 +834,15 @@ func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) {
request = createResourceList(20, 40*GigaByte)
gqm.updateGroupDeltaRequestNoLock("test2-a", request, request)
gqm.updateGroupDeltaUsedNoLock("test2-a", request, createResourceList(0, 0))
assert.Equal(t, request, qi4.CalculateInfo.Request)
assert.Equal(t, request, qi4.CalculateInfo.Used)
assert.Equal(t, request, qi5.CalculateInfo.Request)
assert.Equal(t, request, qi5.CalculateInfo.Used)
assert.Equal(t, v1.ResourceList{}, qi4.CalculateInfo.SelfRequest)
assert.Equal(t, v1.ResourceList{}, qi4.CalculateInfo.SelfUsed)
assert.Equal(t, request, qi5.CalculateInfo.SelfRequest)
assert.Equal(t, request, qi5.CalculateInfo.SelfUsed)

runtime = gqm.RefreshRuntime("test2-a")
assert.Equal(t, request, runtime)

Expand Down Expand Up @@ -1373,15 +1406,23 @@ func TestGroupQuotaManager_UpdatePodCache_UpdatePodIsAssigned_GetPodIsAssigned_U
gqm.updatePodRequestNoLock("1", nil, pod1)
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetRequest())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetNonPreemptibleRequest())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetSelfRequest())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("1").GetSelfNonPreemptibleRequest())
gqm.updatePodUsedNoLock("2", nil, pod1)
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetUsed())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetNonPreemptibleUsed())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetSelfUsed())
assert.Equal(t, createResourceList(10, 10), gqm.GetQuotaInfoByName("2").GetSelfNonPreemptibleUsed())
gqm.updatePodRequestNoLock("1", pod1, nil)
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetRequest())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetNonPreemptibleRequest())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetSelfRequest())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetSelfNonPreemptibleRequest())
gqm.updatePodUsedNoLock("2", pod1, nil)
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetUsed())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetNonPreemptibleUsed())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetSelfUsed())
assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("2").GetSelfNonPreemptibleUsed())
}

func TestGroupQuotaManager_OnPodUpdate(t *testing.T) {
Expand Down Expand Up @@ -1839,3 +1880,50 @@ func TestUpdateQuotaInternalNoLock(t *testing.T) {
assert.Equal(t, createResourceList(25, 25), gqm.GetQuotaInfoByName("1").GetAllocated())
assert.Equal(t, createResourceList(25, 25), gqm.GetQuotaInfoByName("1").GetGuaranteed())
}

func TestUpdateQuotaInternalNoLock_ParenstSelf(t *testing.T) {
gqm := NewGroupQuotaManagerForTest()
gqm.scaleMinQuotaEnabled = true
gqm.UpdateClusterTotalResource(createResourceList(50, 50))

// quota1 Max[40, 40] Min[20,20] request[0,0]
// |-- quota2 Max[30, 20] Min[10,10] request[0,0]
// |-- quota3 Max[20, 10] Min[5,5] request[0,0]
eq1 := createQuota("1", extension.RootQuotaName, 40, 40, 20, 20)
eq2 := createQuota("2", "1", 30, 10, 10, 10)
eq3 := createQuota("3", "1", 20, 10, 5, 5)
eq4 := createQuota("4", "2", 30, 10, 10, 10)
gqm.UpdateQuota(eq1, false)
gqm.UpdateQuota(eq2, false)
gqm.UpdateQuota(eq3, false)
gqm.UpdateQuota(eq4, false)

qi1, qi2, qi3, qi4 := gqm.GetQuotaInfoByName(eq1.Name), gqm.GetQuotaInfoByName(eq2.Name), gqm.GetQuotaInfoByName(eq3.Name), gqm.GetQuotaInfoByName(eq4.Name)

qi2.CalculateInfo.SelfRequest = createResourceList(20, 20)
qi3.CalculateInfo.ChildRequest = createResourceList(20, 20)
qi4.CalculateInfo.ChildRequest = createResourceList(20, 20)
qi2.CalculateInfo.SelfNonPreemptibleRequest = createResourceList(10, 10)
qi3.CalculateInfo.NonPreemptibleRequest = createResourceList(5, 5)
qi4.CalculateInfo.NonPreemptibleRequest = createResourceList(10, 10)

gqm.updateQuotaGroupConfigNoLock()

assert.Equal(t, createResourceList(10, 10), qi4.CalculateInfo.SelfNonPreemptibleRequest)
assert.Equal(t, createResourceList(20, 20), qi4.CalculateInfo.SelfRequest)
assert.Equal(t, createResourceList(5, 5), qi3.CalculateInfo.SelfNonPreemptibleRequest)
assert.Equal(t, createResourceList(20, 20), qi3.CalculateInfo.SelfRequest)
assert.Equal(t, createResourceList(10, 10), qi2.CalculateInfo.SelfNonPreemptibleRequest)
assert.Equal(t, createResourceList(20, 20), qi2.CalculateInfo.SelfRequest)
assert.Equal(t, v1.ResourceList{}, qi1.CalculateInfo.SelfNonPreemptibleRequest)
assert.Equal(t, v1.ResourceList{}, qi1.CalculateInfo.SelfRequest)

assert.Equal(t, createResourceList(10, 10), qi4.CalculateInfo.NonPreemptibleRequest)
assert.Equal(t, createResourceList(20, 20), qi4.CalculateInfo.Request)
assert.Equal(t, createResourceList(5, 5), qi3.CalculateInfo.NonPreemptibleRequest)
assert.Equal(t, createResourceList(20, 20), qi3.CalculateInfo.Request)
assert.Equal(t, createResourceList(20, 20), qi2.CalculateInfo.NonPreemptibleRequest)
assert.Equal(t, createResourceList(40, 30), qi2.CalculateInfo.Request)
assert.Equal(t, createResourceList(25, 25), qi1.CalculateInfo.NonPreemptibleRequest)
assert.Equal(t, createResourceList(50, 20), qi1.CalculateInfo.Request)
}
Loading

0 comments on commit e64e74b

Please sign in to comment.