From 836bff2d14e3ffc0a15c9cfcb1a27c9cc6459087 Mon Sep 17 00:00:00 2001 From: "xulinfei.xlf" Date: Mon, 5 Sep 2022 16:03:56 +0800 Subject: [PATCH] koord-scheduler: add Quota Manager goos: darwin goarch: amd64 cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz BenchmarkGroupQuotaManager_RefreshRuntime-12 6393 211792 ns/op Signed-off-by: xulinfei.xlf --- apis/extension/elastic_quota.go | 13 +- .../elasticquota/core/group_quota_manager.go | 462 +++++++ .../core/group_quota_manager_test.go | 1147 +++++++++++++++++ .../plugins/elasticquota/core/quota_info.go | 117 ++ ...wrapper.go => runtime_quota_calculator.go} | 71 +- ...st.go => runtime_quota_calculator_test.go} | 80 +- .../core/scale_minquota_when_over_root_res.go | 172 +++ .../scale_minquota_when_over_root_res_test.go | 304 +++++ 8 files changed, 2308 insertions(+), 58 deletions(-) create mode 100644 pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go create mode 100644 pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go rename pkg/scheduler/plugins/elasticquota/core/{quota_tree_wrapper.go => runtime_quota_calculator.go} (86%) rename pkg/scheduler/plugins/elasticquota/core/{quota_tree_wrapper_test.go => runtime_quota_calculator_test.go} (86%) create mode 100644 pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go create mode 100644 pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go index 79c2645f6..811cd850e 100644 --- a/apis/extension/elastic_quota.go +++ b/apis/extension/elastic_quota.go @@ -18,8 +18,10 @@ package extension import ( "encoding/json" + "fmt" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apiserver/pkg/quota/v1" "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" ) @@ -59,9 +61,18 @@ func GetSharedWeight(quota *v1alpha1.ElasticQuota) corev1.ResourceList { if exist { resList := corev1.ResourceList{} err := json.Unmarshal([]byte(value), &resList) - if err == nil { + if err == nil && !v1.IsZero(resList) { return resList } } return quota.Spec.Max.DeepCopy() //default equals to max } + +func IsForbiddenModify(quota *v1alpha1.ElasticQuota) (bool, error) { + if quota.Name == SystemQuotaName || quota.Name == RootQuotaName { + // can't modify SystemQuotaGroup + return true, fmt.Errorf("invalid quota %s", quota.Name) + } + + return false, nil +} diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go new file mode 100644 index 000000000..ed4b67e7e --- /dev/null +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go @@ -0,0 +1,462 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "fmt" + "reflect" + "sync" + + v1 "k8s.io/api/core/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" + + "github.com/koordinator-sh/koordinator/apis/extension" +) + +type GroupQuotaManager struct { + // hierarchyUpdateLock used for resourceKeys/quotaInfoMap/quotaTreeWrapper change + hierarchyUpdateLock sync.RWMutex + // totalResource without systemQuotaGroup and DefaultQuotaGroup's used Quota + totalResourceExceptSystemAndDefaultUsed v1.ResourceList + // totalResource with systemQuotaGroup and DefaultQuotaGroup's used Quota + totalResource v1.ResourceList + // resourceKeys helps to store runtimeQuotaCalculators' resourceKey + resourceKeys map[v1.ResourceName]struct{} + // quotaInfoMap stores all the nodes, it can help get all parents conveniently + quotaInfoMap map[string]*QuotaInfo + // runtimeQuotaCalculatorMap helps calculate the subGroups' runtimeQuota in one quotaGroup + runtimeQuotaCalculatorMap map[string]*RuntimeQuotaCalculator + // quotaTopoNodeMap only stores the topology of the quota + quotaTopoNodeMap map[string]*QuotaTopoNode + scaleMinQuotaEnabled bool + // scaleMinQuotaManager is used when overRootResource + scaleMinQuotaManager *ScaleMinQuotaManager + once sync.Once +} + +func NewGroupQuotaManager(systemGroupMax, defaultGroupMax v1.ResourceList) *GroupQuotaManager { + quotaManager := &GroupQuotaManager{ + totalResourceExceptSystemAndDefaultUsed: v1.ResourceList{}, + totalResource: v1.ResourceList{}, + resourceKeys: make(map[v1.ResourceName]struct{}), + quotaInfoMap: make(map[string]*QuotaInfo), + runtimeQuotaCalculatorMap: make(map[string]*RuntimeQuotaCalculator), + quotaTopoNodeMap: make(map[string]*QuotaTopoNode), + scaleMinQuotaManager: NewScaleMinQuotaManager(), + } + quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, "") + quotaManager.quotaInfoMap[extension.SystemQuotaName].setMaxQuotaNoLock(systemGroupMax) + quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, "") + quotaManager.quotaInfoMap[extension.DefaultQuotaName].setMaxQuotaNoLock(defaultGroupMax) + quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName) + return quotaManager +} + +func (gqm *GroupQuotaManager) SetScaleMinQuotaEnabled(flag bool) { + gqm.hierarchyUpdateLock.Lock() + defer gqm.hierarchyUpdateLock.Unlock() + + gqm.scaleMinQuotaEnabled = flag + klog.V(3).Infof("Set ScaleMinQuotaEnabled, flag:%v", gqm.scaleMinQuotaEnabled) +} + +func (gqm *GroupQuotaManager) UpdateClusterTotalResource(deltaRes v1.ResourceList) { + gqm.hierarchyUpdateLock.Lock() + defer gqm.hierarchyUpdateLock.Unlock() + + klog.V(3).Infof("UpdateClusterResource deltaRes:%v", deltaRes) + gqm.updateClusterTotalResourceNoLock(deltaRes) +} + +func (gqm *GroupQuotaManager) updateClusterTotalResourceNoLock(deltaRes v1.ResourceList) { + gqm.totalResource = quotav1.Add(gqm.totalResource, deltaRes) + + sysAndDefaultUsed := gqm.quotaInfoMap[extension.DefaultQuotaName].GetUsed() + sysAndDefaultUsed = quotav1.Add(sysAndDefaultUsed, gqm.quotaInfoMap[extension.SystemQuotaName].GetUsed()) + totalResNoSysOrDefault := quotav1.Subtract(gqm.totalResource, sysAndDefaultUsed) + + diffRes := quotav1.Subtract(totalResNoSysOrDefault, gqm.totalResourceExceptSystemAndDefaultUsed) + + if !quotav1.IsZero(diffRes) { + gqm.totalResourceExceptSystemAndDefaultUsed = totalResNoSysOrDefault.DeepCopy() + gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].SetClusterTotalResource(totalResNoSysOrDefault) + klog.V(3).Infof("UpdateClusterResource finish totalResourceExceptSystemAndDefaultUsed:%v", gqm.totalResourceExceptSystemAndDefaultUsed) + } +} + +func (gqm *GroupQuotaManager) GetClusterTotalResource() v1.ResourceList { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + + return gqm.totalResource.DeepCopy() +} + +func (gqm *GroupQuotaManager) UpdateGroupDeltaRequest(quotaName string, deltaReq v1.ResourceList) { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + + gqm.updateGroupDeltaRequestNoLock(quotaName, deltaReq) +} + +// updateGroupDeltaRequestNoLock no need lock gqm.lock +func (gqm *GroupQuotaManager) updateGroupDeltaRequestNoLock(quotaName string, deltaReq v1.ResourceList) { + curToAllParInfos := gqm.getCurToAllParentGroupQuotaInfoNoLock(quotaName) + allQuotaInfoLen := len(curToAllParInfos) + if allQuotaInfoLen <= 0 { + return + } + + defer gqm.scopedLockForQuotaInfo(curToAllParInfos)() + + gqm.updateGroupDeltaRequestTopoRecursiveNoLock(deltaReq, curToAllParInfos) +} + +// updateGroupDeltaRequestTopoRecursiveNoLock update the quota of a node, also need update all parentNode, the lock operation +// of all quotaInfo is done by gqm. scopedLockForQuotaInfo, so just get treeWrappers' lock when calling treeWrappers' function +func (gqm *GroupQuotaManager) updateGroupDeltaRequestTopoRecursiveNoLock(deltaReq v1.ResourceList, curToAllParInfos []*QuotaInfo) { + for i := 0; i < len(curToAllParInfos); i++ { + curQuotaInfo := curToAllParInfos[i] + directParRuntimeCalculatorPtr := gqm.getRuntimeQuotaCalculatorByNameNoLock(curQuotaInfo.ParentName) + if directParRuntimeCalculatorPtr == nil { + klog.Errorf("treeWrapper not exist! parentName:%v", curQuotaInfo.Name, curQuotaInfo.ParentName) + return + } + oldSubLimitReq := curQuotaInfo.getLimitRequestNoLock() + curQuotaInfo.addRequestNonNegativeNoLock(deltaReq) + newSubLimitReq := curQuotaInfo.getLimitRequestNoLock() + deltaReq = quotav1.Subtract(newSubLimitReq, oldSubLimitReq) + + if directParRuntimeCalculatorPtr.NeedUpdateOneGroupRequest(curQuotaInfo) { + directParRuntimeCalculatorPtr.UpdateOneGroupRequest(curQuotaInfo) + } + } +} + +func (gqm *GroupQuotaManager) UpdateGroupDeltaUsed(quotaName string, delta v1.ResourceList) { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + + gqm.updateGroupDeltaUsedNoLock(quotaName, delta) + + // if systemQuotaGroup or DefaultQuotaGroup's used change, update cluster total resource. + if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName { + gqm.updateClusterTotalResourceNoLock(v1.ResourceList{}) + } +} + +// updateGroupDeltaUsedNoLock updates the usedQuota of a node, it also updates all parent nodes +// no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateGroupDeltaUsedNoLock(quotaName string, delta v1.ResourceList) { + curToAllParInfos := gqm.getCurToAllParentGroupQuotaInfoNoLock(quotaName) + allQuotaInfoLen := len(curToAllParInfos) + if allQuotaInfoLen <= 0 { + return + } + + defer gqm.scopedLockForQuotaInfo(curToAllParInfos)() + for i := 0; i < allQuotaInfoLen; i++ { + quotaInfo := curToAllParInfos[i] + quotaInfo.addUsedNonNegativeNoLock(delta) + } +} + +func (gqm *GroupQuotaManager) RefreshRuntime(quotaName string) v1.ResourceList { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + + return gqm.refreshRuntimeNoLock(quotaName) +} + +func (gqm *GroupQuotaManager) refreshRuntimeNoLock(quotaName string) v1.ResourceList { + quotaInfo := gqm.getQuotaInfoByNameNoLock(quotaName) + if quotaInfo == nil { + return nil + } + + if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName { + return quotaInfo.GetMax() + } + + curToAllParInfos := gqm.getCurToAllParentGroupQuotaInfoNoLock(quotaInfo.Name) + + defer gqm.scopedLockForQuotaInfo(curToAllParInfos)() + + totalRes := gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy() + for i := len(curToAllParInfos) - 1; i >= 0; i-- { + quotaInfo = curToAllParInfos[i] + parRuntimeQuotaCalculator := gqm.getRuntimeQuotaCalculatorByNameNoLock(quotaInfo.ParentName) + if parRuntimeQuotaCalculator == nil { + klog.Errorf("treeWrapper not exist! parentQuotaName:%v", quotaInfo.ParentName) + return nil + } + subTreeWrapper := gqm.getRuntimeQuotaCalculatorByNameNoLock(quotaInfo.Name) + if subTreeWrapper == nil { + klog.Errorf("treeWrapper not exist! parentQuotaName:%v", quotaInfo.Name) + return nil + } + + // 1. execute scaleMin logic with totalRes and update scaledMin if needed + if gqm.scaleMinQuotaEnabled { + needScale, newMinQuota := gqm.scaleMinQuotaManager.GetScaledMinQuota( + totalRes, quotaInfo.ParentName, quotaInfo.Name) + if needScale { + gqm.updateOneGroupAutoScaleMinQuotaNoLock(quotaInfo, newMinQuota) + } + } + + // 2. update parent's runtimeQuota + if quotaInfo.RuntimeVersion != parRuntimeQuotaCalculator.GetVersion() { + parRuntimeQuotaCalculator.UpdateOneGroupRuntimeQuota(quotaInfo) + } + newSubGroupsTotalRes := quotaInfo.CalculateInfo.Runtime.DeepCopy() + + // 3. update subGroup's cluster resource when i >= 1 (still has children) + if i >= 1 { + subTreeWrapper.SetClusterTotalResource(newSubGroupsTotalRes) + } + + // 4. update totalRes + totalRes = newSubGroupsTotalRes + } + + return curToAllParInfos[0].getMaskedRuntimeNoLock() +} + +// updateOneGroupAutoScaleMinQuotaNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateOneGroupAutoScaleMinQuotaNoLock(quotaInfo *QuotaInfo, newMinRes v1.ResourceList) { + if !quotav1.Equals(quotaInfo.CalculateInfo.AutoScaleMin, newMinRes) { + quotaInfo.setAutoScaleMinQuotaNoLock(newMinRes) + gqm.runtimeQuotaCalculatorMap[quotaInfo.ParentName].UpdateOneGroupMinQuota(quotaInfo) + } +} + +func (gqm *GroupQuotaManager) getCurToAllParentGroupQuotaInfoNoLock(quotaName string) []*QuotaInfo { + curToAllParInfos := make([]*QuotaInfo, 0) + quotaInfo := gqm.getQuotaInfoByNameNoLock(quotaName) + if quotaInfo == nil { + return curToAllParInfos + } + + for true { + curToAllParInfos = append(curToAllParInfos, quotaInfo) + if quotaInfo.ParentName == extension.RootQuotaName { + break + } + + quotaInfo = gqm.getQuotaInfoByNameNoLock(quotaInfo.ParentName) + if quotaInfo == nil { + return curToAllParInfos + } + } + + return curToAllParInfos +} + +func (gqm *GroupQuotaManager) GetQuotaInfoByName(quotaName string) *QuotaInfo { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + + return gqm.getQuotaInfoByNameNoLock(quotaName) +} + +func (gqm *GroupQuotaManager) getQuotaInfoByNameNoLock(quotaName string) *QuotaInfo { + return gqm.quotaInfoMap[quotaName] +} + +func (gqm *GroupQuotaManager) getRuntimeQuotaCalculatorByNameNoLock(quotaName string) (treeWrapperPtr *RuntimeQuotaCalculator) { + return gqm.runtimeQuotaCalculatorMap[quotaName] +} + +func (gqm *GroupQuotaManager) scopedLockForQuotaInfo(quotaList []*QuotaInfo) func() { + listLen := len(quotaList) + for i := listLen - 1; i >= 0; i-- { + quotaList[i].lock.Lock() + } + + return func() { + for i := 0; i < listLen; i++ { + quotaList[i].lock.Unlock() + } + } +} + +func (gqm *GroupQuotaManager) UpdateQuota(quota *v1alpha1.ElasticQuota, isDelete bool) error { + gqm.hierarchyUpdateLock.Lock() + defer gqm.hierarchyUpdateLock.Unlock() + + quotaName := quota.Name + if isDelete { + _, exist := gqm.quotaInfoMap[quotaName] + if !exist { + return fmt.Errorf("get quota info failed, quotaName:%v", quotaName) + } + delete(gqm.quotaInfoMap, quotaName) + } else { + newQuotaInfo := NewQuotaInfoFromQuota(quota) + // update the local quotaInfo's crd + if localQuotaInfo, exist := gqm.quotaInfoMap[quotaName]; exist { + localQuotaInfo.UpdateQuotaInfoFromRemote(newQuotaInfo) + } else { + gqm.quotaInfoMap[quotaName] = newQuotaInfo + } + } + gqm.updateQuotaGroupConfigNoLock() + + return nil +} + +func (gqm *GroupQuotaManager) updateQuotaGroupConfigNoLock() { + // rebuild gqm.quotaTopoNodeMap + gqm.buildSubParGroupTopoNoLock() + // reset gqm.runtimeQuotaCalculator + gqm.resetAllGroupQuotaNoLock() +} + +//BuildSubParGroupTopoNoLock reBuild a nodeTree from root, no need to lock gqm.lock +func (gqm *GroupQuotaManager) buildSubParGroupTopoNoLock() { + //rebuild QuotaTopoNodeMap + gqm.quotaTopoNodeMap = make(map[string]*QuotaTopoNode) + rootNode := NewQuotaTopoNode(NewQuotaInfo(false, true, extension.RootQuotaName, extension.RootQuotaName)) + gqm.quotaTopoNodeMap[extension.RootQuotaName] = rootNode + + // add node according to the quotaInfoMap + for quotaName, quotaInfo := range gqm.quotaInfoMap { + if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName { + continue + } + gqm.quotaTopoNodeMap[quotaName] = NewQuotaTopoNode(quotaInfo) + } + + // build tree according to the parGroupName + for _, topoNode := range gqm.quotaTopoNodeMap { + if topoNode.name == extension.RootQuotaName { + continue + } + parQuotaTopoNode := gqm.quotaTopoNodeMap[topoNode.quotaInfo.ParentName] + topoNode.parQuotaTopoNode = parQuotaTopoNode + parQuotaTopoNode.AddChildGroupQuotaInfo(topoNode) + } +} + +// ResetAllGroupQuotaNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() { + childRequestMap, childUsedMap := make(quotaResMapType), make(quotaResMapType) + for quotaName, topoNode := range gqm.quotaTopoNodeMap { + if quotaName == extension.RootQuotaName { + continue + } + topoNode.quotaInfo.lock.Lock() + if !topoNode.quotaInfo.IsParent { + childRequestMap[quotaName] = topoNode.quotaInfo.CalculateInfo.Request.DeepCopy() + childUsedMap[quotaName] = topoNode.quotaInfo.CalculateInfo.Used.DeepCopy() + } + topoNode.quotaInfo.clearForResetNoLock() + topoNode.quotaInfo.lock.Unlock() + } + + // clear old runtimeQuotaCalculator + gqm.runtimeQuotaCalculatorMap = make(map[string]*RuntimeQuotaCalculator) + // reset runtimeQuotaCalculator + gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName) + gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].SetClusterTotalResource(gqm.totalResourceExceptSystemAndDefaultUsed) + rootNode := gqm.quotaTopoNodeMap[extension.RootQuotaName] + gqm.resetAllGroupQuotaRecursiveNoLock(rootNode) + gqm.updateResourceKeyNoLock() + + // subGroup's topo relation may change; refresh the request/used from bottom to top + for quotaName, topoNode := range gqm.quotaTopoNodeMap { + if !topoNode.quotaInfo.IsParent { + gqm.updateGroupDeltaRequestNoLock(quotaName, childRequestMap[quotaName]) + gqm.updateGroupDeltaUsedNoLock(quotaName, childUsedMap[quotaName]) + } + } +} + +// ResetAllGroupQuotaRecursiveNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) resetAllGroupQuotaRecursiveNoLock(rootNode *QuotaTopoNode) { + childGroupQuotaInfos := rootNode.GetChildGroupQuotaInfos() + for subName, topoNode := range childGroupQuotaInfos { + gqm.runtimeQuotaCalculatorMap[subName] = NewRuntimeQuotaCalculator(subName) + + gqm.updateOneGroupMaxQuotaNoLock(topoNode.quotaInfo) + gqm.updateMinQuotaNoLock(topoNode.quotaInfo) + gqm.updateOneGroupSharedWeightNoLock(topoNode.quotaInfo) + + gqm.resetAllGroupQuotaRecursiveNoLock(topoNode) + } +} + +//updateOneGroupMaxQuotaNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateOneGroupMaxQuotaNoLock(quotaInfo *QuotaInfo) { + quotaInfo.lock.Lock() + defer quotaInfo.lock.Unlock() + + runtimeQuotaCalculator := gqm.getRuntimeQuotaCalculatorByNameNoLock(quotaInfo.ParentName) + runtimeQuotaCalculator.UpdateOneGroupMaxQuota(quotaInfo) +} + +// updateMinQuotaNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateMinQuotaNoLock(quotaInfo *QuotaInfo) { + gqm.updateOneGroupOriginalMinQuotaNoLock(quotaInfo) + gqm.scaleMinQuotaManager.Update(quotaInfo.ParentName, quotaInfo.Name, + quotaInfo.CalculateInfo.OriginalMin, gqm.scaleMinQuotaEnabled) +} + +// updateOneGroupOriginalMinQuotaNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateOneGroupOriginalMinQuotaNoLock(quotaInfo *QuotaInfo) { + quotaInfo.lock.Lock() + defer quotaInfo.lock.Unlock() + + quotaInfo.setAutoScaleMinQuotaNoLock(quotaInfo.CalculateInfo.OriginalMin) + gqm.runtimeQuotaCalculatorMap[quotaInfo.ParentName].UpdateOneGroupMinQuota(quotaInfo) +} + +// updateOneGroupSharedWeightNoLock no need to lock gqm.lock +func (gqm *GroupQuotaManager) updateOneGroupSharedWeightNoLock(quotaInfo *QuotaInfo) { + quotaInfo.lock.Lock() + defer quotaInfo.lock.Unlock() + + gqm.runtimeQuotaCalculatorMap[quotaInfo.ParentName].UpdateOneGroupSharedWeight(quotaInfo) +} + +func (gqm *GroupQuotaManager) updateResourceKeyNoLock() { + // collect all dimensions + resourceKeys := make(map[v1.ResourceName]struct{}) + for quotaName, quotaInfo := range gqm.quotaInfoMap { + if quotaName == extension.DefaultQuotaName || quotaName == extension.SystemQuotaName { + continue + } + for resName := range quotaInfo.CalculateInfo.Max { + resourceKeys[resName] = struct{}{} + } + } + + if !reflect.DeepEqual(resourceKeys, gqm.resourceKeys) { + gqm.resourceKeys = resourceKeys + for _, runtimeQuotaCalculator := range gqm.runtimeQuotaCalculatorMap { + runtimeQuotaCalculator.UpdateResourceKeys(resourceKeys) + } + } +} + +func (gqm *GroupQuotaManager) SetQuotaInfoForTest(quotaInfo *QuotaInfo) { + gqm.hierarchyUpdateLock.RLock() + defer gqm.hierarchyUpdateLock.RUnlock() + gqm.quotaInfoMap[quotaInfo.Name] = quotaInfo +} diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go new file mode 100644 index 000000000..5065a46b9 --- /dev/null +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go @@ -0,0 +1,1147 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "encoding/json" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" + + "github.com/koordinator-sh/koordinator/apis/extension" +) + +const ( + GigaByte = 1024 * 1048576 +) + +func TestGroupQuotaManager_QuotaAdd(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.UpdateClusterTotalResource(createResourceList(10000000, 3000*GigaByte)) + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96000, 160*GigaByte, 50000, 50*GigaByte, true, false) + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96000, 160*GigaByte, 80000, 80*GigaByte, true, false) + AddQuotaToManager(t, gqm, "hema", extension.RootQuotaName, 96000, 160*GigaByte, 40000, 40*GigaByte, true, false) + AddQuotaToManager(t, gqm, "kaola", extension.RootQuotaName, 96000, 160*GigaByte, 0, 0*GigaByte, true, false) + AddQuotaToManager(t, gqm, "youku", extension.RootQuotaName, 96000, 160*GigaByte, 96000, 96*GigaByte, true, false) + + quotaInfo := gqm.GetQuotaInfoByName("aliyun") + if quotaInfo.CalculateInfo.SharedWeight.Name(v1.ResourceCPU, resource.DecimalSI).Value() != 96000 { + t.Errorf("error") + } + + assert.Equal(t, 6, len(gqm.runtimeQuotaCalculatorMap)) + assert.Equal(t, 7, len(gqm.quotaInfoMap)) + assert.Equal(t, 2, len(gqm.resourceKeys)) +} + +func TestGroupQuotaManager_QuotaAdd_AutoMakeUpSharedWeight(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.UpdateClusterTotalResource(createResourceList(1000000, 200*GigaByte)) + + quota := CreateQuota("aliyun", extension.RootQuotaName, 96000, 160*GigaByte, 50000, 80*GigaByte, true, false) + delete(quota.Annotations, extension.AnnotationSharedWeight) + + err := gqm.UpdateQuota(quota, false) + if err != nil { + klog.Infof("error") + } + + quotaInfo := gqm.quotaInfoMap["aliyun"] + if quotaInfo.CalculateInfo.SharedWeight.Cpu().Value() != 96000 || quotaInfo.CalculateInfo.SharedWeight.Memory().Value() != 160*GigaByte { + t.Errorf("error:%v", quotaInfo.CalculateInfo.SharedWeight) + } +} + +func TestGroupQuotaManager_QuotaAdd_AutoMakeUpScaleRatio2(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + quota := &v1alpha1.ElasticQuota{ + TypeMeta: metav1.TypeMeta{ + Kind: "Quota", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "", + Annotations: make(map[string]string), + Labels: make(map[string]string), + }, + Spec: v1alpha1.ElasticQuotaSpec{ + Max: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(96, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(1000, resource.DecimalSI), + v1.ResourcePods: *resource.NewQuantity(1000, resource.DecimalSI), + }, + }, + } + + quota.Annotations[extension.AnnotationSharedWeight] = fmt.Sprintf("{\"cpu\":%v, \"memory\":\"%v\"}", 0, 0) + quota.Labels[extension.LabelQuotaParent] = extension.RootQuotaName + quota.Labels[extension.LabelQuotaIsParent] = "false" + gqm.UpdateQuota(quota, false) + + quotaInfo := gqm.GetQuotaInfoByName("test") + if quotaInfo.CalculateInfo.SharedWeight.Cpu().Value() != 96 || + quotaInfo.CalculateInfo.SharedWeight.Memory().Value() != 1000 || + quotaInfo.CalculateInfo.SharedWeight.Pods().Value() != 1000 { + t.Errorf("error:%v", quotaInfo.CalculateInfo.SharedWeight) + } +} + +func TestGroupQuotaManager_UpdateQuotaInternal(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + quota := CreateQuota("aliyun", extension.RootQuotaName, 64, 100*GigaByte, 50, 80*GigaByte, true, false) + gqm.UpdateQuota(quota, false) + quotaInfo := gqm.quotaInfoMap["aliyun"] + assert.True(t, quotaInfo != nil) + assert.False(t, quotaInfo.IsParent) + assert.Equal(t, createResourceList(64, 100*GigaByte), quotaInfo.CalculateInfo.Max) + assert.Equal(t, createResourceList(50, 80*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + assert.Equal(t, int64(64), quotaInfo.CalculateInfo.SharedWeight.Cpu().Value()) + assert.Equal(t, int64(100*GigaByte), quotaInfo.CalculateInfo.SharedWeight.Memory().Value()) + + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, false) + + quota = CreateQuota("aliyun", extension.RootQuotaName, 84, 120*GigaByte, 60, 100*GigaByte, true, false) + quota.Labels[extension.LabelQuotaIsParent] = "true" + err := gqm.UpdateQuota(quota, false) + if err != nil { + klog.Infof("err:%v", err) + } + quotaInfo = gqm.quotaInfoMap["aliyun"] + assert.True(t, quotaInfo != nil) + assert.True(t, quotaInfo.IsParent) + assert.Equal(t, createResourceList(84, 120*GigaByte), quotaInfo.CalculateInfo.Max) + assert.Equal(t, createResourceList(60, 100*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + assert.Equal(t, int64(84), quotaInfo.CalculateInfo.SharedWeight.Cpu().Value()) + assert.Equal(t, int64(120*GigaByte), quotaInfo.CalculateInfo.SharedWeight.Memory().Value()) +} + +func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + setLoglevel("5") + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + // aliyun request[120, 290] runtime == maxQuota + request := createResourceList(120, 290*GigaByte) + gqm.UpdateGroupDeltaRequest("aliyun", request) + runtime := gqm.RefreshRuntime("aliyun") + assert.Equal(t, deltaRes, runtime) + + quota1 := CreateQuota("aliyun", extension.RootQuotaName, 64, 100*GigaByte, 60, 90*GigaByte, true, false) + quota1.Labels[extension.LabelQuotaIsParent] = "false" + err := gqm.UpdateQuota(quota1, false) + if err != nil { + klog.Infof("error:%v", err) + } + quotaInfo := gqm.GetQuotaInfoByName("aliyun") + assert.Equal(t, createResourceList(64, 100*GigaByte), quotaInfo.CalculateInfo.Max) + + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, createResourceList(64, 100*GigaByte), runtime) +} + +func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.UpdateClusterTotalResource(createResourceList(1000, 1000*GigaByte)) + quota1 := AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false) + quota2 := AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, false) + quota3 := AddQuotaToManager(t, gqm, "hema", extension.RootQuotaName, 96, 160*GigaByte, 40, 40*GigaByte, true, false) + assert.Equal(t, 4, len(gqm.runtimeQuotaCalculatorMap)) + assert.Equal(t, 5, len(gqm.quotaInfoMap)) + + //err := gqm.DeleteOneGroup(quota1) + err := gqm.UpdateQuota(quota1, true) + if err != nil { + klog.Infof("err:%v", err) + } + quotaInfo := gqm.GetQuotaInfoByName("aliyun") + assert.True(t, quotaInfo == nil) + + err = gqm.UpdateQuota(quota2, true) + if err != nil { + klog.Infof("err:%v", err) + } + quotaInfo = gqm.GetQuotaInfoByName("alimm") + assert.True(t, quotaInfo == nil) + + err = gqm.UpdateQuota(quota3, true) + if err != nil { + klog.Infof("err:%v", err) + } + quotaInfo = gqm.GetQuotaInfoByName("hema") + assert.True(t, quotaInfo == nil) + + assert.Equal(t, 1, len(gqm.runtimeQuotaCalculatorMap)) + assert.Equal(t, 2, len(gqm.quotaInfoMap)) + + AddQuotaToManager(t, gqm, "youku", extension.RootQuotaName, 96, 160*GigaByte, 70, 70*GigaByte, true, false) + quotaInfo = gqm.GetQuotaInfoByName("youku") + assert.True(t, quotaInfo != nil) + assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap)) + assert.Equal(t, 3, len(gqm.quotaInfoMap)) +} + +func TestGroupQuotaManager_UpdateQuotaDeltaRequest(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false) + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96, 160*GigaByte, 40, 80*GigaByte, true, false) + + // aliyun request[120, 290] runtime == maxQuota + request := createResourceList(120, 200*GigaByte) + gqm.UpdateGroupDeltaRequest("aliyun", request) + runtime := gqm.RefreshRuntime("aliyun") + assert.Equal(t, deltaRes, runtime) + + // alimm request[150, 210] runtime + request = createResourceList(150, 210*GigaByte) + gqm.UpdateGroupDeltaRequest("alimm", request) + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, createResourceList(53, 80*GigaByte), runtime) + runtime = gqm.RefreshRuntime("alimm") + assert.Equal(t, createResourceList(43, 80*GigaByte), runtime) +} + +func TestGroupQuotaManager_NotAllowLentResource(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(100, 0) + gqm.UpdateClusterTotalResource(deltaRes) + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 0, 60, 0, true, false) + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96, 0, 40, 0, false, false) + + request := createResourceList(120, 0) + gqm.UpdateGroupDeltaRequest("aliyun", request) + runtime := gqm.RefreshRuntime("aliyun") + assert.Equal(t, int64(60), runtime.Cpu().Value()) + runtime = gqm.RefreshRuntime("alimm") + assert.Equal(t, int64(40), runtime.Cpu().Value()) +} + +func TestGroupQuotaManager_UpdateQuotaRequest(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false) + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 96, 160*GigaByte, 40, 80*GigaByte, true, false) + + // 1. initial aliyun request [60, 100] + request := createResourceList(60, 100*GigaByte) + gqm.UpdateGroupDeltaRequest("aliyun", request) + runtime := gqm.RefreshRuntime("aliyun") + assert.Equal(t, request, runtime) + + // aliyun request[120, 290] runtime == maxQuota + newRequest := createResourceList(120, 200*GigaByte) + gqm.UpdateGroupDeltaRequest("aliyun", newRequest) + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, deltaRes, runtime) + + // alimm request[150, 210] runtime + request = createResourceList(150, 210*GigaByte) + gqm.UpdateGroupDeltaRequest("alimm", request) + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, createResourceList(53, 80*GigaByte), runtime) + runtime = gqm.RefreshRuntime("alimm") + assert.Equal(t, createResourceList(43, 80*GigaByte), runtime) +} + +func TestGroupQuotaManager_UpdateGroupDeltaUsed(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + AddQuotaToManager(t, gqm, "aliyun", "root", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + // 1. aliyun used[120, 290] runtime == maxQuota + used := createResourceList(120, 290*GigaByte) + gqm.UpdateGroupDeltaUsed("aliyun", used) + quotaInfo := gqm.GetQuotaInfoByName("aliyun") + assert.True(t, quotaInfo != nil) + assert.Equal(t, used, quotaInfo.CalculateInfo.Used) + + // 2. used increases to [130,300] + used = createResourceList(10, 10*GigaByte) + gqm.UpdateGroupDeltaUsed("aliyun", used) + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + assert.True(t, quotaInfo != nil) + assert.Equal(t, createResourceList(130, 300*GigaByte), quotaInfo.CalculateInfo.Used) + + // 3. used decreases to [90,100] + used = createResourceList(-40, -200*GigaByte) + gqm.UpdateGroupDeltaUsed("aliyun", used) + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + assert.True(t, quotaInfo != nil) + assert.Equal(t, createResourceList(90, 100*GigaByte), quotaInfo.CalculateInfo.Used) +} + +func TestGroupQuotaManager_MultiQuotaAdd(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 60, 100*GigaByte, true, true) + AddQuotaToManager(t, gqm, "odps", "aliyun", 96, 160*GigaByte, 10, 30*GigaByte, true, false) + AddQuotaToManager(t, gqm, "holo", "aliyun", 96, 160*GigaByte, 20, 30*GigaByte, true, false) + AddQuotaToManager(t, gqm, "blink", "aliyun", 96, 160*GigaByte, 30, 40*GigaByte, true, false) + + AddQuotaToManager(t, gqm, "alimm", extension.RootQuotaName, 300, 400*GigaByte, 200, 300*GigaByte, true, true) + AddQuotaToManager(t, gqm, "youku", "alimm", 96, 160*GigaByte, 96, 160*GigaByte, true, false) + AddQuotaToManager(t, gqm, "search", "alimm", 82, 100*GigaByte, 80, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "store", "search", 96, 160*GigaByte, 0, 0*GigaByte, true, false) + + assert.Equal(t, 9, len(gqm.runtimeQuotaCalculatorMap)) + assert.Equal(t, 10, len(gqm.quotaInfoMap)) + assert.Equal(t, 2, len(gqm.resourceKeys)) + assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].quotaTree[v1.ResourceCPU].quotaNodes)) + assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap["alimm"].quotaTree[v1.ResourceCPU].quotaNodes)) + assert.Equal(t, 1, len(gqm.runtimeQuotaCalculatorMap["search"].quotaTree[v1.ResourceCPU].quotaNodes)) +} + +// TestGroupQuotaManager_MultiUpdateQuotaRequest test the relationship of request and max +// parentQuotaGroup's request is the sum of limitedRequest of its child. +func TestGroupQuotaManager_MultiUpdateQuotaRequest(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + // aliyun Max[96, 160] Min[50,80] request[96,130] + // |-- yun-a Max[96, 160] Min[50,80] request[96,130] + // |-- a-123 Max[96, 160] Min[50,80] request[96,130] + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "yun-a", "aliyun", 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "a-123", "yun-a", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + request := createResourceList(96, 130*GigaByte) + gqm.UpdateGroupDeltaRequest("a-123", request) + runtime := gqm.RefreshRuntime("a-123") + assert.Equal(t, request, runtime) + runtime = gqm.RefreshRuntime("yun-a") + assert.Equal(t, request, runtime) + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, request, runtime) + + // decrease a-123 Max[64, 128] Min[50,80] request[96,130] + quota1 := CreateQuota("a-123", "yun-a", 64, 128*GigaByte, 50, 80*GigaByte, true, false) + gqm.UpdateQuota(quota1, false) + quotaInfo := gqm.GetQuotaInfoByName("yun-a") + assert.Equal(t, createResourceList(96, 160*GigaByte), quotaInfo.CalculateInfo.Max) + runtime = gqm.RefreshRuntime("a-123") + assert.Equal(t, runtime, quotaInfo.CalculateInfo.Request) + assert.Equal(t, createResourceList(64, 128*GigaByte), runtime) + quotaInfo = gqm.GetQuotaInfoByName("a-123") + assert.Equal(t, request, quotaInfo.CalculateInfo.Request) + + // increase + quota1 = CreateQuota("a-123", "yun-a", 100, 200*GigaByte, 90, 160*GigaByte, true, false) + gqm.UpdateQuota(quota1, false) + quotaInfo = gqm.GetQuotaInfoByName("yun-a") + assert.Equal(t, createResourceList(96, 160*GigaByte), quotaInfo.CalculateInfo.Max) + assert.Equal(t, createResourceList(96, 130*GigaByte), quotaInfo.CalculateInfo.Request) + runtime = gqm.RefreshRuntime("a-123") + assert.Equal(t, runtime, quotaInfo.CalculateInfo.Request) + assert.Equal(t, request, runtime) + quotaInfo = gqm.GetQuotaInfoByName("a-123") + assert.Equal(t, request, quotaInfo.CalculateInfo.Request) +} + +func TestGroupQuotaManager_UpdateDefaultQuotaGroup(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + gqm.UpdateQuota(&v1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: v1alpha1.ElasticQuotaSpec{ + Max: createResourceList(100, 1000), + }, + }, false) + assert.Equal(t, gqm.GetQuotaInfoByName("default").CalculateInfo.Max, createResourceList(100, 1000)) + runtime := gqm.RefreshRuntime("default") + assert.Equal(t, runtime, createResourceList(100, 1000)) +} + +// TestGroupQuotaManager_MultiUpdateQuotaRequest2 test the relationship of min/max and request +// increase the request to test the case request < min, min < request < max, max < request +func TestGroupQuotaManager_MultiUpdateQuotaRequest2(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + // aliyun Max[96, 160] Min[80,80] + // |-- yun-a Max[60, 120] Min[50,80] + // |-- a-123 Max[30, 60] Min[20,40] + AddQuotaToManager(t, gqm, "aliyun", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "yun-a", "aliyun", 60, 80*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "a-123", "yun-a", 30, 60*GigaByte, 20, 40*GigaByte, true, false) + + // a-123 request[10,30] request < min + request := createResourceList(10, 30*GigaByte) + gqm.UpdateGroupDeltaRequest("a-123", request) + runtime := gqm.RefreshRuntime("a-123") + assert.Equal(t, request, runtime) + runtime = gqm.RefreshRuntime("yun-a") + assert.Equal(t, request, runtime) + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, request, runtime) + + // a-123 add request[15,15] totalRequest[25,45] request > min + request = createResourceList(15, 15*GigaByte) + gqm.UpdateGroupDeltaRequest("a-123", request) + runtime = gqm.RefreshRuntime("a-123") + assert.Equal(t, createResourceList(25, 45*GigaByte), runtime) + quotaInfo := gqm.GetQuotaInfoByName("yun-a") + assert.Equal(t, createResourceList(25, 45*GigaByte), quotaInfo.CalculateInfo.Request) + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + assert.Equal(t, createResourceList(25, 45*GigaByte), quotaInfo.CalculateInfo.Request) + + // a-123 add request[30,30] totalRequest[55,75] request > max + request = createResourceList(30, 30*GigaByte) + gqm.UpdateGroupDeltaRequest("a-123", request) + runtime = gqm.RefreshRuntime("a-123") + assert.Equal(t, createResourceList(30, 60*GigaByte), runtime) + quotaInfo = gqm.GetQuotaInfoByName("yun-a") + assert.Equal(t, createResourceList(30, 60*GigaByte), quotaInfo.CalculateInfo.Request) + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + assert.Equal(t, createResourceList(30, 60*GigaByte), quotaInfo.CalculateInfo.Request) +} + +// TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota1 test scaledMinQuota when quotaGroup's sum of the minQuota +// is larger than totalRes; and enlarge the totalRes to test whether gqm can recover the oriMinQuota or not. +func TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota1(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + + AddQuotaToManager(t, gqm, "p", "root", 1000, 1000*GigaByte, 300, 300*GigaByte, true, true) + AddQuotaToManager(t, gqm, "a", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + AddQuotaToManager(t, gqm, "b", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + AddQuotaToManager(t, gqm, "c", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + + request := createResourceList(200, 200*GigaByte) + gqm.UpdateGroupDeltaRequest("a", request) + gqm.UpdateGroupDeltaRequest("b", request) + gqm.UpdateGroupDeltaRequest("c", request) + + deltaRes := createResourceList(200, 200*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + + runtime := gqm.RefreshRuntime("p") + assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) + gqm.RefreshRuntime("a") + gqm.RefreshRuntime("b") + gqm.RefreshRuntime("c") + runtime = gqm.RefreshRuntime("a") + assert.Equal(t, createResourceList(67, 200*GigaByte/3+1), runtime) + // after group "c" refreshRuntime, the runtime of "a" and "b" has changed + assert.Equal(t, createResourceList(67, 200*GigaByte/3+1), gqm.RefreshRuntime("a")) + assert.Equal(t, createResourceList(67, 200*GigaByte/3+1), gqm.RefreshRuntime("b")) + + quotaInfo := gqm.GetQuotaInfoByName("p") + assert.Equal(t, createResourceList(200, 200*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("a") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("b") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("c") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) + + // large + deltaRes = createResourceList(400, 400*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + + runtime = gqm.RefreshRuntime("p") + assert.Equal(t, createResourceList(600, 600*GigaByte), runtime) + + runtime = gqm.RefreshRuntime("a") + assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) + + runtime = gqm.RefreshRuntime("b") + assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) + + runtime = gqm.RefreshRuntime("c") + assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) + + quotaInfo = gqm.GetQuotaInfoByName("p") + assert.Equal(t, createResourceList(300, 300*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("a") + assert.Equal(t, createResourceList(100, 100*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("b") + assert.Equal(t, createResourceList(100, 100*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("c") + assert.Equal(t, createResourceList(100, 100*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) +} + +// TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota1 test scaledMinQuota when quotaGroup's sum of the +// minQuota is larger than totalRes, with one of the quotaGroup's request is zero. +func TestGroupQuotaManager_MultiUpdateQuotaRequest_WithScaledMinQuota2(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + deltaRes := createResourceList(1, 1*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + + AddQuotaToManager(t, gqm, "p", "root", 1000, 1000*GigaByte, 300, 300*GigaByte, true, true) + AddQuotaToManager(t, gqm, "a", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + AddQuotaToManager(t, gqm, "b", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + AddQuotaToManager(t, gqm, "c", "p", 1000, 1000*GigaByte, 100, 100*GigaByte, true, false) + + request := createResourceList(200, 200*GigaByte) + gqm.UpdateGroupDeltaRequest("a", request) + gqm.UpdateGroupDeltaRequest("b", createResourceList(0, 0)) + gqm.UpdateGroupDeltaRequest("c", request) + gqm.UpdateClusterTotalResource(createResourceList(199, 199*GigaByte)) + runtime := gqm.RefreshRuntime("p") + assert.Equal(t, createResourceList(200, 200*GigaByte), runtime) + + gqm.RefreshRuntime("a") + gqm.RefreshRuntime("b") + gqm.RefreshRuntime("c") + + runtime = gqm.RefreshRuntime("a") + assert.Equal(t, createResourceList(100, 100*GigaByte), runtime) + + runtime = gqm.RefreshRuntime("b") + assert.Equal(t, createResourceList(0, 0*GigaByte), runtime) + + runtime = gqm.RefreshRuntime("c") + assert.Equal(t, createResourceList(100, 100*GigaByte), runtime) + + quotaInfo := gqm.GetQuotaInfoByName("p") + assert.Equal(t, createResourceList(200, 200*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("a") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("b") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) + + quotaInfo = gqm.GetQuotaInfoByName("c") + assert.Equal(t, createResourceList(66, 200*GigaByte/3), quotaInfo.CalculateInfo.AutoScaleMin) +} + +func TestGroupQuotaManager_MultiUpdateQuotaUsed(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + AddQuotaToManager(t, gqm, "aliyun", "root", 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "yun-1", "aliyun", 96, 160*GigaByte, 50, 80*GigaByte, true, true) + AddQuotaToManager(t, gqm, "yun1-1", "yun-1", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + used := createResourceList(120, 290*GigaByte) + gqm.UpdateGroupDeltaUsed("yun1-1", used) + quotaInfo := gqm.GetQuotaInfoByName("yun1-1") + assert.True(t, quotaInfo != nil) + assert.Equal(t, used, quotaInfo.CalculateInfo.Used) + + quotaInfo = gqm.GetQuotaInfoByName("yun-1") + assert.True(t, quotaInfo != nil) + assert.Equal(t, used, quotaInfo.CalculateInfo.Used) + + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + assert.True(t, quotaInfo != nil) + assert.Equal(t, used, quotaInfo.CalculateInfo.Used) +} + +func TestGroupQuotaManager_UpdateQuotaParentName(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + assert.Equal(t, deltaRes, gqm.totalResource) + totalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, deltaRes, totalRes) + + // alimm Max[96, 160] Min[50,80] request[20,40] + // `-- mm-a Max[96, 160] Min[50,80] request[20,40] + // aliyun Max[96, 160] Min[50,80] request[60,100] + // `-- yun-a Max[96, 160] Min[50,80] request[60,100] + // `-- a-123 Max[96, 160] Min[50,80] request[60,100] + AddQuotaToManager(t, gqm, "aliyun", "root", 96, 160*GigaByte, 100, 160*GigaByte, true, true) + AddQuotaToManager(t, gqm, "yun-a", "aliyun", 96, 160*GigaByte, 50, 80*GigaByte, true, true) + changeQuota := AddQuotaToManager(t, gqm, "a-123", "yun-a", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + AddQuotaToManager(t, gqm, "alimm", "root", 96, 160*GigaByte, 100, 160*GigaByte, true, true) + AddQuotaToManager(t, gqm, "mm-a", "alimm", 96, 160*GigaByte, 50, 80*GigaByte, true, false) + + // a-123 request [60,100] + request := createResourceList(60, 100*GigaByte) + gqm.UpdateGroupDeltaRequest("a-123", request) + gqm.UpdateGroupDeltaUsed("a-123", request) + runtime := gqm.RefreshRuntime("a-123") + assert.Equal(t, request, runtime) + + runtime = gqm.RefreshRuntime("yun-a") + assert.Equal(t, request, runtime) + + runtime = gqm.RefreshRuntime("aliyun") + assert.Equal(t, request, runtime) + + // mm-a request [20,40] + request = createResourceList(20, 40*GigaByte) + gqm.UpdateGroupDeltaRequest("mm-a", request) + gqm.UpdateGroupDeltaUsed("mm-a", request) + runtime = gqm.RefreshRuntime("mm-a") + assert.Equal(t, request, runtime) + + runtime = gqm.RefreshRuntime("alimm") + assert.Equal(t, request, runtime) + + // a-123 mv alimm + // alimm Max[96, 160] Min[100,160] request[80,140] + // `-- mm-a Max[96, 160] Min[50,80] request[20,40] + // `-- a-123 Max[96, 160] Min[50,80] request[60,100] + // aliyun Max[96, 160] Min[100,160] request[0,0] + // `-- yun-a Max[96, 160] Min[50,80] request[0,0] + changeQuota.Labels[extension.LabelQuotaParent] = "alimm" + gqm.UpdateQuota(changeQuota, false) + + quotaInfo := gqm.GetQuotaInfoByName("yun-a") + gqm.RefreshRuntime("yun-a") + assert.Equal(t, v1.ResourceList{}, quotaInfo.CalculateInfo.Request) + assert.Equal(t, v1.ResourceList{}, quotaInfo.CalculateInfo.Used) + assert.Equal(t, createResourceList(0, 0), quotaInfo.CalculateInfo.Runtime) + + quotaInfo = gqm.GetQuotaInfoByName("aliyun") + gqm.RefreshRuntime("aliyun") + assert.Equal(t, v1.ResourceList{}, quotaInfo.CalculateInfo.Request) + assert.Equal(t, v1.ResourceList{}, quotaInfo.CalculateInfo.Used) + assert.Equal(t, createResourceList(0, 0), quotaInfo.CalculateInfo.Runtime) + + quotaInfo = gqm.GetQuotaInfoByName("a-123") + gqm.RefreshRuntime("a-123") + assert.Equal(t, createResourceList(60, 100*GigaByte), quotaInfo.CalculateInfo.Request) + assert.Equal(t, createResourceList(60, 100*GigaByte), quotaInfo.CalculateInfo.Used) + assert.Equal(t, createResourceList(60, 100*GigaByte), quotaInfo.CalculateInfo.Runtime) + assert.Equal(t, "alimm", quotaInfo.ParentName) + + quotaInfo = gqm.GetQuotaInfoByName("mm-a") + gqm.RefreshRuntime("mm-a") + assert.Equal(t, createResourceList(20, 40*GigaByte), quotaInfo.CalculateInfo.Request) + assert.Equal(t, createResourceList(20, 40*GigaByte), quotaInfo.CalculateInfo.Used) + assert.Equal(t, createResourceList(20, 40*GigaByte), quotaInfo.CalculateInfo.Runtime) + + quotaInfo = gqm.GetQuotaInfoByName("alimm") + gqm.RefreshRuntime("alimm") + assert.Equal(t, createResourceList(80, 140*GigaByte), quotaInfo.CalculateInfo.Request) + assert.Equal(t, createResourceList(80, 140*GigaByte), quotaInfo.CalculateInfo.Used) + assert.Equal(t, createResourceList(80, 140*GigaByte), quotaInfo.CalculateInfo.Runtime) +} + +func TestGroupQuotaManager_UpdateClusterTotalResource(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + totalRes := createResourceList(96, 160*GigaByte) + gqm.UpdateClusterTotalResource(totalRes) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, totalRes, gqm.totalResourceExceptSystemAndDefaultUsed) + quotaTotalRes := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, totalRes, quotaTotalRes) + + totalRes = createResourceList(64, 360*GigaByte) + gqm.UpdateClusterTotalResource(createResourceList(-32, 200*GigaByte)) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, totalRes, gqm.totalResourceExceptSystemAndDefaultUsed) + quotaTotalRes = gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, totalRes, quotaTotalRes) + + totalRes = createResourceList(100, 540*GigaByte) + gqm.UpdateClusterTotalResource(createResourceList(36, 180*GigaByte)) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, totalRes, gqm.totalResourceExceptSystemAndDefaultUsed) + quotaTotalRes = gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, totalRes, quotaTotalRes) + + sysUsed := createResourceList(10, 30*GigaByte) + gqm.UpdateGroupDeltaUsed(extension.SystemQuotaName, sysUsed) + assert.Equal(t, sysUsed, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetUsed()) + + // 90, 510 + delta := totalRes.DeepCopy() + delta = quotav1.SubtractWithNonNegativeResult(delta, sysUsed) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, delta, gqm.totalResourceExceptSystemAndDefaultUsed) + quotaTotalRes = gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource.DeepCopy() + assert.Equal(t, delta, quotaTotalRes) + + // 80, 480 + gqm.UpdateGroupDeltaUsed(extension.SystemQuotaName, createResourceList(10, 30)) + delta = quotav1.Subtract(delta, createResourceList(10, 30)) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, delta, gqm.totalResourceExceptSystemAndDefaultUsed) + + // 70, 450 + defaultUsed := createResourceList(10, 30) + gqm.UpdateGroupDeltaUsed(extension.DefaultQuotaName, defaultUsed) + assert.Equal(t, defaultUsed, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetUsed()) + delta = quotav1.Subtract(delta, defaultUsed) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, delta, gqm.totalResourceExceptSystemAndDefaultUsed) + + // 60 420 + gqm.UpdateGroupDeltaUsed(extension.DefaultQuotaName, defaultUsed) + delta = quotav1.Subtract(delta, defaultUsed) + assert.Equal(t, totalRes, gqm.totalResource) + assert.Equal(t, delta, gqm.totalResourceExceptSystemAndDefaultUsed) +} + +func AddQuotaToManager(t *testing.T, + gqm *GroupQuotaManager, + quotaName string, + parent string, + maxCpu, maxMem, minCpu, minMem int64, allowLentResource bool, isParent bool) *v1alpha1.ElasticQuota { + quota := CreateQuota(quotaName, parent, maxCpu, maxMem, minCpu, minMem, allowLentResource, isParent) + err := gqm.UpdateQuota(quota, false) + if err != nil { + klog.Infof("error:%v", err) + } + quotaInfo := gqm.quotaInfoMap[quotaName] + assert.True(t, quotaInfo != nil) + assert.Equal(t, quotaName, quotaInfo.Name) + assert.Equal(t, quotaInfo.CalculateInfo.Max, createResourceList(maxCpu, maxMem)) + assert.Equal(t, quotaInfo.CalculateInfo.OriginalMin, createResourceList(minCpu, minMem)) + assert.Equal(t, quotaInfo.CalculateInfo.SharedWeight.Memory().Value(), maxMem) + assert.Equal(t, quotaInfo.CalculateInfo.SharedWeight.Cpu().Value(), maxCpu) + + //assert whether the parent quotaTree has the sub quota group + quotaTreeWrapper := gqm.runtimeQuotaCalculatorMap[quotaName] + assert.True(t, quotaTreeWrapper != nil) + quotaTreeWrapper = gqm.runtimeQuotaCalculatorMap[parent] + assert.True(t, quotaTreeWrapper != nil) + find, nodeInfo := quotaTreeWrapper.quotaTree[v1.ResourceCPU].find(quotaName) + assert.True(t, find) + assert.Equal(t, nodeInfo.quotaName, quotaName) + return quota +} + +func CreateQuota(quotaName string, parent string, maxCpu, maxMem, minCpu, minMem int64, allowLentResource bool, isParent bool) *v1alpha1.ElasticQuota { + quota := &v1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: quotaName, + Annotations: make(map[string]string), + Labels: make(map[string]string), + }, + Spec: v1alpha1.ElasticQuotaSpec{ + Max: createResourceList(maxCpu, maxMem), + Min: createResourceList(minCpu, minMem), + }, + } + + quota.Annotations[extension.AnnotationSharedWeight] = fmt.Sprintf("{\"cpu\":%v, \"memory\":\"%v\"}", maxCpu, maxMem) + quota.Labels[extension.LabelQuotaParent] = parent + if allowLentResource { + quota.Labels[extension.LabelAllowLentResource] = "true" + } else { + quota.Labels[extension.LabelAllowLentResource] = "false" + } + + if isParent { + quota.Labels[extension.LabelQuotaIsParent] = "true" + } else { + quota.Labels[extension.LabelQuotaIsParent] = "false" + } + return quota +} + +func TestGroupQuotaManager_MultiChildMaxGreaterParentMax_MaxGreaterThanTotalRes(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(300, 8000) + gqm.UpdateClusterTotalResource(deltaRes) + + // odps Max[600, 4096] Min[100, 100] + // |-- admin Max[500, 2048] Min[100, 100] Req[500, 4096] + AddQuotaToManager(t, gqm, "odps", "root", 600, 4096, 100, 100, true, true) + AddQuotaToManager(t, gqm, "admin", "odps", 500, 2048, 100, 100, true, false) + + // admin Request [500, 4096] limitRequest [500, 2048] + // odps Request [500,2048] limitedRequest [500, 2048] limited by rootRes [300, 8000] -> [300,2048] + request := createResourceList(500, 4096) + gqm.UpdateGroupDeltaRequest("admin", request) + runtime := gqm.RefreshRuntime("admin") + assert.Equal(t, createResourceList(300, 2048), runtime) + fmt.Printf("quota1 runtime:%v\n", runtime) + + // admin Request [1050, 8192] limitRequest [500, 2048] + // odps Request [500,2048] limitedRequest [500, 2048] limited by rootRes [300, 8000] -> [300,2048] + request = createResourceList(550, 4096) + gqm.UpdateGroupDeltaRequest("admin", request) + runtime = gqm.RefreshRuntime("admin") + fmt.Printf("quota1 runtime:%v\n", runtime) + + quotaInfo := gqm.GetQuotaInfoByName("odps") + assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(500, 2048)) + assert.Equal(t, quotaInfo.getLimitRequestNoLock(), createResourceList(500, 2048)) + assert.Equal(t, quotaInfo.CalculateInfo.Max, createResourceList(600, 4096)) + assert.Equal(t, quotaInfo.CalculateInfo.Runtime, createResourceList(300, 2048)) + quotaInfo = gqm.GetQuotaInfoByName("admin") + assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(1050, 8192)) + assert.Equal(t, quotaInfo.getLimitRequestNoLock(), createResourceList(500, 2048)) + assert.Equal(t, quotaInfo.CalculateInfo.Max, createResourceList(500, 2048)) + assert.Equal(t, quotaInfo.CalculateInfo.Runtime, createResourceList(300, 2048)) +} + +func TestGroupQuotaManager_MultiChildMaxGreaterParentMax(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + deltaRes := createResourceList(350, 1800*GigaByte) + gqm.UpdateClusterTotalResource(deltaRes) + + // odps Max[300, 1024] Min[176, 756] + // |-- admin Max[500, 2048] Min[100, 512] + AddQuotaToManager(t, gqm, "odps", "root", 300, 1024*GigaByte, 176, 756*GigaByte, true, true) + AddQuotaToManager(t, gqm, "admin", "odps", 500, 2048*GigaByte, 100, 512*GigaByte, true, false) + + // odps max < request < admin max + // admin Request[400, 1500] limitedRequest [400, 1500] + // odps Request [400,1500] limitedRequest [300, 1024] + request := createResourceList(400, 1500*GigaByte) + gqm.UpdateGroupDeltaRequest("admin", request) + quotaInfo := gqm.GetQuotaInfoByName("odps") + assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(400, 1500*GigaByte)) + quotaInfo = gqm.GetQuotaInfoByName("admin") + assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(400, 1500*GigaByte)) + runtime := gqm.RefreshRuntime("admin") + assert.Equal(t, createResourceList(300, 1024*GigaByte), runtime) + fmt.Printf("quota1 runtime:%v\n", runtime) + + // odps max < admin max < request + // admin Request[800, 3000] limitedRequest [500, 2048] + // odps Request [500, 2048] limitedRequest [300, 1024] + gqm.UpdateGroupDeltaRequest("admin", request) + runtime = gqm.RefreshRuntime("admin") + assert.Equal(t, createResourceList(300, 1024*GigaByte), runtime) + fmt.Printf("quota1 runtime:%v\n", runtime) +} + +func setLoglevel(logLevel string) { + var level klog.Level + if err := level.Set(logLevel); err != nil { + fmt.Printf("failed set klog.logging.verbosity %v: %v", 5, err) + } + fmt.Printf("successfully set klog.logging.verbosity to %v", 5) +} + +func TestGroupQuotaManager_UpdateQuotaTreeDimension_UpdateQuota(t *testing.T) { + setLoglevel("5") + gqm := NewGroupQuotaManager4Test() + + info3 := CreateQuota("3", extension.RootQuotaName, 1000, 10000, 100, 1000, true, false) + info3.Spec.Max["tmp"] = *resource.NewQuantity(1, resource.DecimalSI) + res := createResourceList(1000, 10000) + gqm.UpdateClusterTotalResource(res) + err := gqm.UpdateQuota(info3, false) + if err != nil { + klog.Infof("%v", err) + } + assert.Equal(t, len(gqm.resourceKeys), 3) +} + +func createQuota(name, parent string, cpuMax, memMax, cpuMin, memMin int64) *v1alpha1.ElasticQuota { + eq := CreateQuota(name, parent, cpuMax, memMax, cpuMin, memMin, true, false) + return eq +} + +func TestGroupQuotaManager_RefreshAndGetRuntimeQuota_UpdateQuota(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + + gqm.scaleMinQuotaEnabled = true + cluRes := createResourceList(50, 50) + gqm.UpdateClusterTotalResource(cluRes) + + qi1 := createQuota("1", extension.RootQuotaName, 40, 40, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 0, 0) + + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + + assert.Equal(t, len(gqm.resourceKeys), 2) + //case1: there is no request, runtime quota is zero + runtime := gqm.RefreshRuntime("1") + assert.Equal(t, runtime, createResourceList(0, 0)) + + //case2: no existed group + assert.Nil(t, gqm.RefreshRuntime("5")) + gqm.UpdateGroupDeltaRequest("1", createResourceList(5, 5)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(5, 5)) + gq1 := gqm.GetQuotaInfoByName("1") + gq2 := gqm.GetQuotaInfoByName("2") + + //case3: version is same, should not update + gq1.RuntimeVersion = gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].globalRuntimeVersion + gq2.RuntimeVersion = gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].globalRuntimeVersion + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(0, 0)) + assert.Equal(t, gqm.RefreshRuntime("2"), v1.ResourceList{}) + + //case4: version is different, should update + gq1.RuntimeVersion = 0 + gq2.RuntimeVersion = 0 + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(5, 5)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(5, 5)) + + //case5: request is larger than min + gqm.UpdateGroupDeltaRequest("1", createResourceList(25, 25)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(25, 25)) + //1 min [10,10] -> toPartitionRes [40,40] -> runtime [30,30] + //2 min [0,0] -> toPartitionRes [40,40] -> runtime [20,20] + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(30, 30)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(20, 20)) +} + +func TestGroupQuotaManager_UpdateSharedWeight_UpdateQuota(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + + gqm.UpdateClusterTotalResource(createResourceList(60, 60)) + + //case1: if not config SharedWeight, equal to maxQuota + qi1 := createQuota("1", extension.RootQuotaName, 40, 40, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 0, 0) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + + assert.Equal(t, gqm.quotaInfoMap["1"].CalculateInfo.SharedWeight.Cpu().Value(), int64(40)) + assert.Equal(t, gqm.quotaInfoMap["2"].CalculateInfo.SharedWeight.Cpu().Value(), int64(40)) + + //case2: if config SharedWeight + sharedWeight1, _ := json.Marshal(createResourceList(30, 30)) + qi1.Annotations[extension.AnnotationSharedWeight] = string(sharedWeight1) + sharedWeight2, _ := json.Marshal(createResourceList(10, 10)) + qi2.Annotations[extension.AnnotationSharedWeight] = string(sharedWeight2) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + result1, _ := json.Marshal(gqm.quotaInfoMap["1"].CalculateInfo.SharedWeight) + result2, _ := json.Marshal(gqm.quotaInfoMap["2"].CalculateInfo.SharedWeight) + assert.Equal(t, result1, sharedWeight1) + assert.Equal(t, result2, sharedWeight2) +} + +func TestGroupQuotaManager_UpdateOneGroupMaxQuota_UpdateQuota(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + + gqm.UpdateClusterTotalResource(createResourceList(50, 50)) + + qi1 := createQuota("1", extension.RootQuotaName, 40, 40, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 10, 10) + + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + + //case1: min < req < max + gqm.UpdateGroupDeltaRequest("1", createResourceList(35, 35)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(35, 35)) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(25, 25)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(25, 25)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(35, 35)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(35, 35)) + + //case2: decrease max, min < max < request + qi1.Spec.Max = createResourceList(30, 30) + qi2.Spec.Max = createResourceList(30, 30) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(25, 25)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(25, 25)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(30, 30)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(30, 30)) + + //case3: increase max, min < req < max + qi1.Spec.Max = createResourceList(50, 50) + qi2.Spec.Max = createResourceList(50, 50) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(25, 25)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(25, 25)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(35, 35)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(35, 35)) +} + +func TestGroupQuotaManager_UpdateOneGroupMinQuota_UpdateQuota(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + + gqm.UpdateClusterTotalResource(createResourceList(50, 50)) + + qi1 := createQuota("1", extension.RootQuotaName, 40, 40, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 10, 10) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + + //case1: min < req < max + gqm.UpdateGroupDeltaRequest("1", createResourceList(15, 15)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(15, 15)) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(15, 15)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(15, 15)) + + //case2: increase min, req = min + qi1.Spec.Min = createResourceList(15, 15) + qi2.Spec.Min = createResourceList(15, 15) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(15, 15)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(15, 15)) + + //case3: increase min, req < min + qi1.Spec.Min = createResourceList(20, 20) + qi2.Spec.Min = createResourceList(20, 20) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(15, 15)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(15, 15)) + + //case4: decrease min, min < req < max + qi1.Spec.Min = createResourceList(5, 5) + qi2.Spec.Min = createResourceList(5, 5) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(15, 15)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(15, 15)) + + //case5: update min, min == max, request > max + qi1.Spec.Min = createResourceList(40, 40) + qi2.Spec.Min = createResourceList(5, 5) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + gqm.UpdateGroupDeltaRequest("1", createResourceList(100, 100)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(100, 100)) + assert.Equal(t, gqm.RefreshRuntime("1"), createResourceList(40, 40)) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(10, 10)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["1"], createResourceList(40, 40)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].groupReqLimit["2"], createResourceList(40, 40)) +} + +func TestGroupQuotaManager_DeleteOneGroup_UpdateQuota(t *testing.T) { + gqm := NewGroupQuotaManager4Test() + gqm.scaleMinQuotaEnabled = true + + gqm.UpdateClusterTotalResource(createResourceList(50, 50)) + + qi1 := createQuota("1", extension.RootQuotaName, 40, 40, 10, 10) + q1 := CreateQuota("1", extension.RootQuotaName, 40, 40, 10, 10, true, false) + qi2 := createQuota("2", extension.RootQuotaName, 40, 40, 10, 10) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + gqm.UpdateGroupDeltaRequest("1", createResourceList(15, 15)) + gqm.UpdateGroupDeltaRequest("2", createResourceList(15, 15)) + + // delete one group + gqm.UpdateQuota(q1, true) + assert.Equal(t, gqm.RefreshRuntime("2"), createResourceList(15, 15)) + assert.Equal(t, gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].totalResource, createResourceList(50, 50)) + assert.Nil(t, gqm.quotaInfoMap["1"]) +} + +func NewGroupQuotaManager4Test() *GroupQuotaManager { + quotaManager := &GroupQuotaManager{ + totalResourceExceptSystemAndDefaultUsed: v1.ResourceList{}, + totalResource: v1.ResourceList{}, + resourceKeys: make(map[v1.ResourceName]struct{}), + quotaInfoMap: make(map[string]*QuotaInfo), + runtimeQuotaCalculatorMap: make(map[string]*RuntimeQuotaCalculator), + scaleMinQuotaManager: NewScaleMinQuotaManager(), + quotaTopoNodeMap: make(map[string]*QuotaTopoNode), + } + quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, "") + quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, "") + quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName) + return quotaManager +} + +func BenchmarkGroupQuotaManager_RefreshRuntime(b *testing.B) { + b.StopTimer() + gqm := NewGroupQuotaManager4Test() + random := rand.New(rand.NewSource(time.Now().UnixNano())) + for i := 0; i < 2000; i++ { + AddQuotaToManager2(gqm, fmt.Sprintf("%v", i), extension.RootQuotaName, 96, 160*GigaByte, 0, 0, true, false) + } + + totalReqMem, totalReqCpu := float64(0), float64(0) + for j := 0; j < 2000; j++ { + reqMem := int64(random.Int() % 2000) + reqCpu := int64(random.Int() % 2000) + request := createResourceList(reqCpu, reqMem) + totalReqMem += float64(reqMem) + totalReqCpu += float64(reqCpu) + gqm.UpdateGroupDeltaRequest(fmt.Sprintf("%v", j), request) + } + totalRes := createResourceList(int64(totalReqCpu/1.5), int64(totalReqMem/1.5)) + gqm.UpdateClusterTotalResource(totalRes) + b.StartTimer() + for i := 0; i < b.N; i++ { + gqm.RefreshRuntime("0") + gqm.getQuotaInfoByNameNoLock("0").RuntimeVersion = 0 + } +} + +func AddQuotaToManager2(gqm *GroupQuotaManager, quotaName string, parent string, + maxCpu, maxMem, minCpu, minMem int64, allowLentResource bool, isParent bool) *v1alpha1.ElasticQuota { + quota := CreateQuota(quotaName, parent, maxCpu, maxMem, minCpu, minMem, allowLentResource, isParent) + gqm.UpdateQuota(quota, false) + return quota +} diff --git a/pkg/scheduler/plugins/elasticquota/core/quota_info.go b/pkg/scheduler/plugins/elasticquota/core/quota_info.go index 56c8fb852..9796ca5f9 100644 --- a/pkg/scheduler/plugins/elasticquota/core/quota_info.go +++ b/pkg/scheduler/plugins/elasticquota/core/quota_info.go @@ -79,6 +79,52 @@ func NewQuotaInfo(isParent, allowLentResource bool, name, parentName string) *Qu }, } } +func (qi *QuotaInfo) DeepCopy() *QuotaInfo { + if qi == nil { + return nil + } + qi.lock.Lock() + defer qi.lock.Unlock() + + return &QuotaInfo{ + Name: qi.Name, + ParentName: qi.ParentName, + IsParent: qi.IsParent, + AllowLentResource: qi.AllowLentResource, + RuntimeVersion: qi.RuntimeVersion, + CalculateInfo: QuotaCalculateInfo{ + Max: qi.CalculateInfo.Max.DeepCopy(), + AutoScaleMin: qi.CalculateInfo.AutoScaleMin.DeepCopy(), + OriginalMin: qi.CalculateInfo.OriginalMin.DeepCopy(), + Used: qi.CalculateInfo.Used.DeepCopy(), + Request: qi.CalculateInfo.Request.DeepCopy(), + SharedWeight: qi.CalculateInfo.SharedWeight.DeepCopy(), + Runtime: qi.CalculateInfo.Runtime.DeepCopy(), + }, + } +} + +// UpdateQuotaInfoFromRemote the CRD(max/oriMin/sharedWeight/allowLentResource/isParent/ParentName) of the quota maybe changed, +// so need update localQuotaInfo's information from inputQuotaInfo. +func (qi *QuotaInfo) UpdateQuotaInfoFromRemote(quotaInfo *QuotaInfo) { + qi.lock.Lock() + defer qi.lock.Unlock() + + if quotaInfo.Name == extension.SystemQuotaName { + return + } + + qi.setMaxQuotaNoLock(quotaInfo.CalculateInfo.Max) + qi.setOriginalMinQuotaNoLock(quotaInfo.CalculateInfo.OriginalMin) + sharedWeight := quotaInfo.CalculateInfo.SharedWeight.DeepCopy() + if quotav1.IsZero(sharedWeight) { + sharedWeight = quotaInfo.CalculateInfo.Max.DeepCopy() + } + qi.CalculateInfo.SharedWeight = sharedWeight + qi.AllowLentResource = quotaInfo.AllowLentResource + qi.IsParent = quotaInfo.IsParent + qi.ParentName = quotaInfo.ParentName +} // getLimitRequestNoLock returns the min value of request and max, as max is the quotaGroup's upper limit of resources. // As the multi-hierarchy quota Model described in the PR, when passing a request upwards, passing a request exceeding its @@ -105,6 +151,13 @@ func (qi *QuotaInfo) addRequestNonNegativeNoLock(delta v1.ResourceList) { } } +func (qi *QuotaInfo) addUsedNonNegativeNoLock(delta v1.ResourceList) { + qi.CalculateInfo.Used = quotav1.Add(qi.CalculateInfo.Used, delta) + for _, resName := range quotav1.IsNegative(qi.CalculateInfo.Used) { + qi.CalculateInfo.Used[resName] = *resource.NewQuantity(0, resource.DecimalSI) + } +} + func (qi *QuotaInfo) setMaxQuotaNoLock(res v1.ResourceList) { qi.CalculateInfo.Max = res.DeepCopy() } @@ -121,6 +174,30 @@ func (qi *QuotaInfo) setSharedWeightNoLock(res v1.ResourceList) { qi.CalculateInfo.SharedWeight = res.DeepCopy() } +func (qi *QuotaInfo) GetRequest() v1.ResourceList { + qi.lock.Lock() + defer qi.lock.Unlock() + return qi.CalculateInfo.Request.DeepCopy() +} + +func (qi *QuotaInfo) GetUsed() v1.ResourceList { + qi.lock.Lock() + defer qi.lock.Unlock() + return qi.CalculateInfo.Used.DeepCopy() +} + +func (qi *QuotaInfo) GetRuntime() v1.ResourceList { + qi.lock.Lock() + defer qi.lock.Unlock() + return qi.CalculateInfo.Runtime.DeepCopy() +} + +func (qi *QuotaInfo) GetMax() v1.ResourceList { + qi.lock.Lock() + defer qi.lock.Unlock() + return qi.CalculateInfo.Max.DeepCopy() +} + func NewQuotaInfoFromQuota(quota *v1alpha1.ElasticQuota) *QuotaInfo { isParent := extension.IsParentQuota(quota) parentName := extension.GetParentQuotaName(quota) @@ -135,3 +212,43 @@ func NewQuotaInfoFromQuota(quota *v1alpha1.ElasticQuota) *QuotaInfo { return quotaInfo } + +func (qi *QuotaInfo) getMaskedRuntimeNoLock() v1.ResourceList { + return quotav1.Mask(qi.CalculateInfo.Runtime, quotav1.ResourceNames(qi.CalculateInfo.Max)) +} + +func (qi *QuotaInfo) clearForResetNoLock() { + qi.CalculateInfo.Request = v1.ResourceList{} + qi.CalculateInfo.Used = v1.ResourceList{} + qi.CalculateInfo.Runtime = v1.ResourceList{} + qi.RuntimeVersion = 0 +} + +// QuotaTopoNode only contains the topology of the parent/child relationship, +// helps to reconstruct quotaTree from the rootQuotaGroup to all the leafQuotaNode. +type QuotaTopoNode struct { + name string + quotaInfo *QuotaInfo + parQuotaTopoNode *QuotaTopoNode + childGroupQuotaInfos map[string]*QuotaTopoNode +} + +func NewQuotaTopoNode(quotaInfo *QuotaInfo) *QuotaTopoNode { + return &QuotaTopoNode{ + name: quotaInfo.Name, + quotaInfo: quotaInfo, // not deepCopy + childGroupQuotaInfos: make(map[string]*QuotaTopoNode), + } +} + +func (qtn *QuotaTopoNode) AddChildGroupQuotaInfo(childNode *QuotaTopoNode) { + qtn.childGroupQuotaInfos[childNode.name] = childNode +} + +func (qtn *QuotaTopoNode) GetChildGroupQuotaInfos() map[string]*QuotaTopoNode { + group := make(map[string]*QuotaTopoNode) + for key, v := range qtn.childGroupQuotaInfos { + group[key] = v + } + return group +} diff --git a/pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper.go b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go similarity index 86% rename from pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper.go rename to pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go index 8579e73a3..bb91bd703 100644 --- a/pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper.go +++ b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go @@ -159,9 +159,9 @@ func (qt *quotaTree) iterationForRedistribution(totalRes, totalSharedWeight int6 type quotaResMapType map[string]v1.ResourceList type quotaTreeMapType map[v1.ResourceName]*quotaTree -// QuotaTreeWrapper helps to calculate the childGroups' all resource dimensions' runtimeQuota of the +// RuntimeQuotaCalculator helps to calculate the childGroups' all resource dimensions' runtimeQuota of the // corresponding quotaInfo(treeName) -type QuotaTreeWrapper struct { +type RuntimeQuotaCalculator struct { globalRuntimeVersion int64 // increase as the runtimeQuota changed resourceKeys map[v1.ResourceName]struct{} // the resource dimensions groupReqLimit quotaResMapType // all childQuotaInfos' limitedRequest @@ -171,8 +171,8 @@ type QuotaTreeWrapper struct { treeName string // the same as the parentQuotaInfo's Name } -func NewQuotaTreeWrapper(treeName string) *QuotaTreeWrapper { - return &QuotaTreeWrapper{ +func NewRuntimeQuotaCalculator(treeName string) *RuntimeQuotaCalculator { + return &RuntimeQuotaCalculator{ globalRuntimeVersion: 1, resourceKeys: make(map[v1.ResourceName]struct{}), groupReqLimit: make(quotaResMapType), @@ -182,7 +182,7 @@ func NewQuotaTreeWrapper(treeName string) *QuotaTreeWrapper { } } -func (qtw *QuotaTreeWrapper) UpdateResourceKeys(resourceKeys map[v1.ResourceName]struct{}) { +func (qtw *RuntimeQuotaCalculator) UpdateResourceKeys(resourceKeys map[v1.ResourceName]struct{}) { newResourceKey := make(map[v1.ResourceName]struct{}) for resKey := range resourceKeys { newResourceKey[resKey] = struct{}{} @@ -195,7 +195,7 @@ func (qtw *QuotaTreeWrapper) UpdateResourceKeys(resourceKeys map[v1.ResourceName qtw.updateQuotaTreeDimensionByResourceKeysNoLock() } -func (qtw *QuotaTreeWrapper) updateQuotaTreeDimensionByResourceKeysNoLock() { +func (qtw *RuntimeQuotaCalculator) updateQuotaTreeDimensionByResourceKeysNoLock() { //lock outside for resKey := range qtw.quotaTree { if _, exist := qtw.resourceKeys[resKey]; !exist { @@ -210,8 +210,8 @@ func (qtw *QuotaTreeWrapper) updateQuotaTreeDimensionByResourceKeysNoLock() { } } -// DeleteOneGroup delete a childGroup, should delete both the quotaTree and groupReqLimit, then adjustQuota to refresh runtimeQuota. -func (qtw *QuotaTreeWrapper) DeleteOneGroup(quotaInfo *QuotaInfo) { +// DeleteOneGroup delete a childGroup, should delete both the quotaTree and groupReqLimit, then increase globalRuntimeVersion. +func (qtw *RuntimeQuotaCalculator) DeleteOneGroup(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -220,7 +220,7 @@ func (qtw *QuotaTreeWrapper) DeleteOneGroup(quotaInfo *QuotaInfo) { } delete(qtw.groupReqLimit, quotaInfo.Name) - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("DeleteOneGroup finish", quotaInfo) @@ -228,9 +228,9 @@ func (qtw *QuotaTreeWrapper) DeleteOneGroup(quotaInfo *QuotaInfo) { } // UpdateOneGroupMaxQuota updates a childGroup's maxQuota, the limitedReq of the quotaGroup may change, so -// should update reqLimit in the process, then adjustQuota to refresh runtimeQuota. +// should update reqLimit in the process, then increase globalRuntimeVersion // need use newMaxQuota to adjust dimension. -func (qtw *QuotaTreeWrapper) UpdateOneGroupMaxQuota(quotaInfo *QuotaInfo) { +func (qtw *RuntimeQuotaCalculator) UpdateOneGroupMaxQuota(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -260,15 +260,15 @@ func (qtw *QuotaTreeWrapper) UpdateOneGroupMaxQuota(quotaInfo *QuotaInfo) { localReqLimit[resKey] = reqLimitPerKey } - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("UpdateOneGroupMaxQuota finish", quotaInfo) } } -// UpdateOneGroupMinQuota the autoScaleMin change, need adjustQuota to refresh runtime. -func (qtw *QuotaTreeWrapper) UpdateOneGroupMinQuota(quotaInfo *QuotaInfo) { +// UpdateOneGroupMinQuota the autoScaleMin change, then increase globalRuntimeVersion +func (qtw *RuntimeQuotaCalculator) UpdateOneGroupMinQuota(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -287,15 +287,15 @@ func (qtw *QuotaTreeWrapper) UpdateOneGroupMinQuota(quotaInfo *QuotaInfo) { } } - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("UpdateOneGroupMinQuota finish", quotaInfo) } } -// UpdateOneGroupSharedWeight , the ability to share the "lent to" resource change, need adjustQuota to refresh runtime. -func (qtw *QuotaTreeWrapper) UpdateOneGroupSharedWeight(quotaInfo *QuotaInfo) { +// UpdateOneGroupSharedWeight , the ability to share the "lent to" resource change, then increase globalRuntimeVersion +func (qtw *RuntimeQuotaCalculator) UpdateOneGroupSharedWeight(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -314,7 +314,7 @@ func (qtw *QuotaTreeWrapper) UpdateOneGroupSharedWeight(quotaInfo *QuotaInfo) { } } - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("UpdateOneGroupSharedWeight finish", quotaInfo) @@ -324,7 +324,7 @@ func (qtw *QuotaTreeWrapper) UpdateOneGroupSharedWeight(quotaInfo *QuotaInfo) { // NeedUpdateOneGroupRequest if oldReqLimit is the same as newReqLimit, no need to adjustQuota. // the request of one group may change frequently, but the cost of adjustQuota is high, so here // need to judge whether you need to update QuotaNode's request or not. -func (qtw *QuotaTreeWrapper) NeedUpdateOneGroupRequest(quotaInfo *QuotaInfo) bool { +func (qtw *RuntimeQuotaCalculator) NeedUpdateOneGroupRequest(quotaInfo *QuotaInfo) bool { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -340,7 +340,8 @@ func (qtw *QuotaTreeWrapper) NeedUpdateOneGroupRequest(quotaInfo *QuotaInfo) boo return false } -func (qtw *QuotaTreeWrapper) UpdateOneGroupRequest(quotaInfo *QuotaInfo) { +// UpdateOneGroupRequest the request of one group change, need increase globalRuntimeVersion +func (qtw *RuntimeQuotaCalculator) UpdateOneGroupRequest(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() @@ -363,22 +364,22 @@ func (qtw *QuotaTreeWrapper) UpdateOneGroupRequest(quotaInfo *QuotaInfo) { reqLimit[resKey] = reqLimitPerKey } - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("UpdateOneGroupRequest finish", quotaInfo) } } -// SetClusterTotalResource increase/decrease the totalResource of the quotaTreeWrapper, the resource that can be "lent to" will -// change, need adjustQuota to refresh runtime. -func (qtw *QuotaTreeWrapper) SetClusterTotalResource(full v1.ResourceList) { +// SetClusterTotalResource increase/decrease the totalResource of the RuntimeQuotaCalculator, the resource that can be "lent to" will +// change, then increase globalRuntimeVersion +func (qtw *RuntimeQuotaCalculator) SetClusterTotalResource(full v1.ResourceList) { qtw.lock.Lock() defer qtw.lock.Unlock() oldTotalRes := qtw.totalResource.DeepCopy() qtw.totalResource = full.DeepCopy() - qtw.adjustQuotaNoLock() + qtw.globalRuntimeVersion++ klog.V(5).Infof("UpdateClusterTotalResource"+ "treeName:%v oldTotalResource:%v newTotalResource:%v reqLimit:%v refreshedVersion:%v", @@ -386,23 +387,29 @@ func (qtw *QuotaTreeWrapper) SetClusterTotalResource(full v1.ResourceList) { } // UpdateOneGroupRuntimeQuota update the quotaInfo's runtimeQuota as the quotaNode's runtime. -func (qtw *QuotaTreeWrapper) UpdateOneGroupRuntimeQuota(quotaInfo *QuotaInfo) { +func (qtw *RuntimeQuotaCalculator) UpdateOneGroupRuntimeQuota(quotaInfo *QuotaInfo) { qtw.lock.Lock() defer qtw.lock.Unlock() + if quotaInfo.RuntimeVersion == qtw.globalRuntimeVersion { + return + } + + qtw.calculateRuntimeNoLock() + for resKey := range qtw.resourceKeys { if exist, quotaNode := qtw.quotaTree[resKey].find(quotaInfo.Name); exist { quotaInfo.CalculateInfo.Runtime[resKey] = *resource.NewQuantity(quotaNode.runtimeQuota, resource.DecimalSI) } } - quotaInfo.RuntimeVersion = qtw.globalRuntimeVersion + if klog.V(5).Enabled() { qtw.logQuotaInfoNoLock("UpdateOneGroupRuntimeQuota finish", quotaInfo) } } -func (qtw *QuotaTreeWrapper) getGroupRequestLimitNoLock(quotaName string) v1.ResourceList { +func (qtw *RuntimeQuotaCalculator) getGroupRequestLimitNoLock(quotaName string) v1.ResourceList { res, exist := qtw.groupReqLimit[quotaName] if !exist { res = v1.ResourceList{} @@ -411,23 +418,21 @@ func (qtw *QuotaTreeWrapper) getGroupRequestLimitNoLock(quotaName string) v1.Res return res } -func (qtw *QuotaTreeWrapper) GetVersion() int64 { +func (qtw *RuntimeQuotaCalculator) GetVersion() int64 { qtw.lock.Lock() defer qtw.lock.Unlock() return qtw.globalRuntimeVersion } -func (qtw *QuotaTreeWrapper) adjustQuotaNoLock() { +func (qtw *RuntimeQuotaCalculator) calculateRuntimeNoLock() { //lock outside for resKey := range qtw.resourceKeys { totalResourcePerKey := *qtw.totalResource.Name(resKey, resource.DecimalSI) qtw.quotaTree[resKey].redistribution(totalResourcePerKey.Value()) } - - qtw.globalRuntimeVersion++ } -func (qtw *QuotaTreeWrapper) logQuotaInfoNoLock(verb string, quotaInfo *QuotaInfo) { +func (qtw *RuntimeQuotaCalculator) logQuotaInfoNoLock(verb string, quotaInfo *QuotaInfo) { klog.Infof("%s\n"+ "quotaName:%v quotaParentName:%v IsParent:%v request:%v maxQuota:%v OriginalMinQuota:%v"+ "autoScaleMinQuota:%v SharedWeight:%v runtime:%v used:%v treeName:%v totalResource:%v reqLimit:%v refreshedVersion:%v", verb, diff --git a/pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper_test.go b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go similarity index 86% rename from pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper_test.go rename to pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go index 46f115f24..e52ae3509 100644 --- a/pkg/scheduler/plugins/elasticquota/core/quota_tree_wrapper_test.go +++ b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go @@ -49,6 +49,20 @@ func TestQuotaInfo_GetLimitRequest(t *testing.T) { assertObj.Equal(*resource.NewQuantity(2000, resource.DecimalSI), quotaInfo.getLimitRequestNoLock()[corev1.ResourceMemory]) } +func TestQuotaInfo_AddRequestNonNegativeNoLock(t *testing.T) { + req1 := createResourceList(-100, -100) + quotaInfo := &QuotaInfo{ + CalculateInfo: QuotaCalculateInfo{ + Request: createResourceList(50, 50), + Used: createResourceList(40, 40), + }, + } + quotaInfo.addRequestNonNegativeNoLock(req1) + quotaInfo.addUsedNonNegativeNoLock(req1) + assert.Equal(t, quotaInfo.CalculateInfo.Request, createResourceList(0, 0)) + assert.Equal(t, quotaInfo.CalculateInfo.Used, createResourceList(0, 0)) +} + func TestNewQuotaInfoFromQuota(t *testing.T) { eQ := createElasticQuota() quotaInfo := NewQuotaInfoFromQuota(eQ) @@ -108,8 +122,8 @@ func createElasticQuota() *v1alpha1.ElasticQuota { return eQ } -func TestQuotaTreeWrapper_Iteration4AdjustQuota(t *testing.T) { - qtw := NewQuotaTreeWrapper("testTreeName") +func TestRuntimeQuotaCalculator_Iteration4AdjustQuota(t *testing.T) { + qtw := NewRuntimeQuotaCalculator("testTreeName") resourceKey := make(map[corev1.ResourceName]struct{}) cpu := corev1.ResourceCPU resourceKey[cpu] = struct{}{} @@ -120,7 +134,7 @@ func TestQuotaTreeWrapper_Iteration4AdjustQuota(t *testing.T) { qtw.quotaTree[cpu].insert("node4", 80, 70, 15, true) qtw.totalResource = corev1.ResourceList{} qtw.totalResource[corev1.ResourceCPU] = *resource.NewQuantity(100, resource.DecimalSI) - qtw.adjustQuotaNoLock() + qtw.calculateRuntimeNoLock() if qtw.globalRuntimeVersion == 0 { t.Error("error") } @@ -141,8 +155,8 @@ func createQuotaInfoWithRes(name string, max, min corev1.ResourceList) *QuotaInf return quotaInfo } -func createQuotaTreeWrapper() *QuotaTreeWrapper { - qtw := NewQuotaTreeWrapper("0") +func createRuntimeQuotaCalculator() *RuntimeQuotaCalculator { + qtw := NewRuntimeQuotaCalculator("0") resKeys := make(map[corev1.ResourceName]struct{}) resKeys[corev1.ResourceCPU] = struct{}{} resKeys[corev1.ResourceMemory] = struct{}{} @@ -151,9 +165,9 @@ func createQuotaTreeWrapper() *QuotaTreeWrapper { return qtw } -func TestQuotaTreeWrapper_UpdateResourceKeys(t *testing.T) { +func TestRuntimeQuotaCalculator_UpdateResourceKeys(t *testing.T) { assertObj := assert.New(t) - qtw := NewQuotaTreeWrapper("0") + qtw := NewRuntimeQuotaCalculator("0") resKeys := make(map[corev1.ResourceName]struct{}) resKeys[corev1.ResourceCPU] = struct{}{} resKeys[corev1.ResourceMemory] = struct{}{} @@ -180,11 +194,11 @@ func TestQuotaTreeWrapper_UpdateResourceKeys(t *testing.T) { assertObj.True(exist, "update quota tree failed") } -func TestQuotaTreeWrapper_DeleteOneGroup(t *testing.T) { +func TestRuntimeQuotaCalculator_DeleteOneGroup(t *testing.T) { max := createResourceList(100, 1000) min := createResourceList(70, 7000) quotaInfo := createQuotaInfoWithRes("aliyun", max, min) - qtw := createQuotaTreeWrapper() + qtw := createRuntimeQuotaCalculator() quotaInfo.setMaxQuotaNoLock(max) qtw.UpdateOneGroupMaxQuota(quotaInfo) @@ -198,11 +212,11 @@ func TestQuotaTreeWrapper_DeleteOneGroup(t *testing.T) { assert.Equal(t, 0, len(qtw.quotaTree["cpu"].quotaNodes)) } -func TestQuotaTreeWrapper_UpdateOneGroupMaxQuota(t *testing.T) { +func TestRuntimeQuotaCalculator_UpdateOneGroupMaxQuota(t *testing.T) { max := createResourceList(100, 1000) min := createResourceList(70, 7000) quotaInfo := createQuotaInfoWithRes("aliyun", max, min) - qtw := createQuotaTreeWrapper() + qtw := createRuntimeQuotaCalculator() quotaInfo.setMaxQuotaNoLock(max) qtw.UpdateOneGroupMaxQuota(quotaInfo) @@ -227,7 +241,7 @@ func TestQuotaTreeWrapper_UpdateOneGroupMaxQuota(t *testing.T) { assert.Equal(t, max, qtw.totalResource) } -func TestQuotaTreeWrapper_UpdateOneGroupMinQuota(t *testing.T) { +func TestRuntimeQuotaCalculator_UpdateOneGroupMinQuota(t *testing.T) { assertObj := assert.New(t) max := createResourceList(100, 10000) minQuota := createResourceList(70, 7000) @@ -235,7 +249,7 @@ func TestQuotaTreeWrapper_UpdateOneGroupMinQuota(t *testing.T) { // totalRequest = request = min, totalResource = max quotaInfo.CalculateInfo.Request = minQuota.DeepCopy() - qtw := createQuotaTreeWrapper() + qtw := createRuntimeQuotaCalculator() qtw.groupReqLimit[quotaInfo.Name] = minQuota qtw.SetClusterTotalResource(max) quotaInfo.setAutoScaleMinQuotaNoLock(minQuota) @@ -244,6 +258,7 @@ func TestQuotaTreeWrapper_UpdateOneGroupMinQuota(t *testing.T) { assertObj.Equal(2, len(qtw.resourceKeys)) assertObj.Equal(max.Name(corev1.ResourceCPU, resource.DecimalSI), qtw.totalResource.Name(corev1.ResourceCPU, resource.DecimalSI)) assertObj.Equal(max.Name(corev1.ResourceMemory, resource.DecimalSI), qtw.totalResource.Name(corev1.ResourceMemory, resource.DecimalSI)) + qtw.UpdateOneGroupRuntimeQuota(quotaInfo) assertObj.Equal(qtw.quotaTree["cpu"].quotaNodes[quotaInfo.Name].runtimeQuota, int64(70)) assertObj.Equal(qtw.quotaTree["memory"].quotaNodes[quotaInfo.Name].runtimeQuota, int64(7000)) assertObj.Equal(qtw.quotaTree["cpu"].quotaNodes[quotaInfo.Name].min, int64(70)) @@ -251,16 +266,17 @@ func TestQuotaTreeWrapper_UpdateOneGroupMinQuota(t *testing.T) { newMin := createResourceList(50, 5000) quotaInfo.setAutoScaleMinQuotaNoLock(newMin) qtw.UpdateOneGroupMinQuota(quotaInfo) + qtw.UpdateOneGroupRuntimeQuota(quotaInfo) assertObj.Equal(qtw.quotaTree["cpu"].quotaNodes[quotaInfo.Name].runtimeQuota, int64(70)) assertObj.Equal(qtw.quotaTree["memory"].quotaNodes[quotaInfo.Name].runtimeQuota, int64(7000)) assertObj.Equal(qtw.quotaTree["cpu"].quotaNodes[quotaInfo.Name].min, int64(50)) } -func TestQuotaTreeWrapper_UpdateOneGroupSharedWeight(t *testing.T) { +func TestRuntimeQuotaCalculator_UpdateOneGroupSharedWeight(t *testing.T) { max := createResourceList(100, 1000) min := createResourceList(70, 7000) quotaInfo := createQuotaInfoWithRes("aliyun", max, min) - qtw := createQuotaTreeWrapper() + qtw := createRuntimeQuotaCalculator() qtw.UpdateOneGroupSharedWeight(quotaInfo) maxCpu := max["cpu"] @@ -279,11 +295,11 @@ func TestQuotaTreeWrapper_UpdateOneGroupSharedWeight(t *testing.T) { assert.Equal(t, sharedCpu.Value(), qtw.quotaTree["cpu"].quotaNodes[quotaInfo.Name].sharedWeight) } -func TestQuotaTreeWrapper_NeedUpdateOneGroupRequest(t *testing.T) { +func TestRuntimeQuotaCalculator_NeedUpdateOneGroupRequest(t *testing.T) { max := createResourceList(100, 1000) min := createResourceList(70, 7000) quotaInfo := createQuotaInfoWithRes("aliyun", max, min) - qtw := createQuotaTreeWrapper() + qtw := createRuntimeQuotaCalculator() update := qtw.NeedUpdateOneGroupRequest(quotaInfo) assert.False(t, update) @@ -293,8 +309,8 @@ func TestQuotaTreeWrapper_NeedUpdateOneGroupRequest(t *testing.T) { assert.True(t, update) } -func TestQuotaTreeWrapper_UpdateOneGroupRequest(t *testing.T) { - qtw := createQuotaTreeWrapper() +func TestRuntimeQuotaCalculator_UpdateOneGroupRequest(t *testing.T) { + qtw := createRuntimeQuotaCalculator() totalResource := createResourceList(50, 5000) qtw.SetClusterTotalResource(totalResource) quotaCount := 5 @@ -320,8 +336,8 @@ func TestQuotaTreeWrapper_UpdateOneGroupRequest(t *testing.T) { } } -func TestQuotaTreeWrapper_UpdateOneGroupRuntimeQuota(t *testing.T) { - qtw := createQuotaTreeWrapper() +func TestRuntimeQuotaCalculator_UpdateOneGroupRuntimeQuota(t *testing.T) { + qtw := createRuntimeQuotaCalculator() totalResource := createResourceList(100, 1000) qtw.SetClusterTotalResource(totalResource) @@ -373,8 +389,8 @@ func TestQuotaTreeWrapper_UpdateOneGroupRuntimeQuota(t *testing.T) { assert.Equal(t, aliYun.CalculateInfo.AutoScaleMin, aliYun.CalculateInfo.Runtime) } -func TestQuotaTreeWrapper_UpdateOneGroupRuntimeQuota2(t *testing.T) { - qtw := createQuotaTreeWrapper() +func TestRuntimeQuotaCalculator_UpdateOneGroupRuntimeQuota2(t *testing.T) { + qtw := createRuntimeQuotaCalculator() totalResource := createResourceList(120, 1200) qtw.SetClusterTotalResource(totalResource) @@ -407,7 +423,7 @@ func TestQuotaTreeWrapper_UpdateOneGroupRuntimeQuota2(t *testing.T) { assert.Equal(t, aliYun.CalculateInfo.Runtime, createResourceList(60, 600)) } -func updateQuotaInfo(wrapper *QuotaTreeWrapper, info *QuotaInfo, max, min, sharedWeight corev1.ResourceList) { +func updateQuotaInfo(wrapper *RuntimeQuotaCalculator, info *QuotaInfo, max, min, sharedWeight corev1.ResourceList) { info.setMaxQuotaNoLock(max) wrapper.UpdateOneGroupMaxQuota(info) info.setAutoScaleMinQuotaNoLock(min) @@ -415,3 +431,19 @@ func updateQuotaInfo(wrapper *QuotaTreeWrapper, info *QuotaInfo, max, min, share info.setSharedWeightNoLock(sharedWeight) wrapper.UpdateOneGroupSharedWeight(info) } + +func TestQuotaInfo_GetRuntime(t *testing.T) { + qi := &QuotaInfo{ + Name: "3", + CalculateInfo: QuotaCalculateInfo{ + Max: createResourceList(100, 200), + Runtime: corev1.ResourceList{ + "GPU": *resource.NewQuantity(20, resource.DecimalSI), + "cpu": *resource.NewQuantity(10, resource.DecimalSI), + }, + }, + } + assert.Equal(t, qi.getMaskedRuntimeNoLock(), corev1.ResourceList{ + "cpu": *resource.NewQuantity(10, resource.DecimalSI), + }) +} diff --git a/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go new file mode 100644 index 000000000..42a19333e --- /dev/null +++ b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go @@ -0,0 +1,172 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "sync" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + "k8s.io/klog/v2" +) + +// ScaleMinQuotaManager The child nodes under each node will be divided into two categories, one allows +// scaling min quota, and the other does not allow. When our total resources are insufficient, the min quota of child +// nodes will be proportionally reduced. In order to calculate the scaling of the min quota, we will count the sum of +// the min quota of the child nodes of each node in advance. The quota of the child nodes that allow scaling is stored +// in enableScaleSubsSumMinQuotaMap, and the quota of child nodes that do not allow scaling is stored in +// disableScaleSubsSumMinQuotaMap +type ScaleMinQuotaManager struct { + lock sync.RWMutex + // enableScaleSubsSumMinQuotaMap key: quotaName, val: sum of its enableScale children's minQuota + enableScaleSubsSumMinQuotaMap map[string]v1.ResourceList + // disableScaleSubsSumMinQuotaMap key: quotaName, val: sum of its disableScale children's minQuota + disableScaleSubsSumMinQuotaMap map[string]v1.ResourceList + // originalMinQuotaMap stores the original minQuota, when children's sum minQuota is smaller than the + // totalRes, just return the originalMinQuota. + originalMinQuotaMap map[string]v1.ResourceList + quotaEnableMinQuotaScaleMap map[string]bool +} + +func NewScaleMinQuotaManager() *ScaleMinQuotaManager { + info := &ScaleMinQuotaManager{ + originalMinQuotaMap: make(map[string]v1.ResourceList), + enableScaleSubsSumMinQuotaMap: make(map[string]v1.ResourceList), + disableScaleSubsSumMinQuotaMap: make(map[string]v1.ResourceList), + quotaEnableMinQuotaScaleMap: make(map[string]bool), + } + return info +} + +func (s *ScaleMinQuotaManager) Update(parQuotaName, subQuotaName string, subMinQuota v1.ResourceList, enableScaleMinQuota bool) { + s.lock.Lock() + defer s.lock.Unlock() + + if _, ok := s.enableScaleSubsSumMinQuotaMap[parQuotaName]; !ok { + s.enableScaleSubsSumMinQuotaMap[parQuotaName] = v1.ResourceList{} + } + if _, ok := s.disableScaleSubsSumMinQuotaMap[parQuotaName]; !ok { + s.disableScaleSubsSumMinQuotaMap[parQuotaName] = v1.ResourceList{} + } + + // step1: delete the oldMinQuota if present + if enable, ok := s.quotaEnableMinQuotaScaleMap[subQuotaName]; ok { + if enable { + s.enableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult(s.enableScaleSubsSumMinQuotaMap[parQuotaName], + s.originalMinQuotaMap[subQuotaName]) + } else { + s.disableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult(s.disableScaleSubsSumMinQuotaMap[parQuotaName], + s.originalMinQuotaMap[subQuotaName]) + } + } + + // step2: add the newMinQuota + if enableScaleMinQuota { + s.enableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.Add(s.enableScaleSubsSumMinQuotaMap[parQuotaName], subMinQuota) + } else { + s.disableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.Add(s.disableScaleSubsSumMinQuotaMap[parQuotaName], subMinQuota) + } + + klog.V(5).Infof("UpdateScaleMinQuota, quota:%v originalMinQuota change from :%v to %v,"+ + "enableMinQuotaScale change from :%v to :%v", subQuotaName, s.originalMinQuotaMap[subQuotaName], + subMinQuota, s.quotaEnableMinQuotaScaleMap[subQuotaName], enableScaleMinQuota) + + // step3: record the newMinQuota + s.originalMinQuotaMap[subQuotaName] = subMinQuota + s.quotaEnableMinQuotaScaleMap[subQuotaName] = enableScaleMinQuota +} + +func (s *ScaleMinQuotaManager) Delete(parQuotaName, subQuotaName string) { + s.lock.Lock() + defer s.lock.Unlock() + + if s.quotaEnableMinQuotaScaleMap[subQuotaName] == true { + if _, ok := s.enableScaleSubsSumMinQuotaMap[parQuotaName]; ok { + s.enableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult( + s.enableScaleSubsSumMinQuotaMap[parQuotaName], s.originalMinQuotaMap[subQuotaName]) + } + } else { + if _, ok := s.disableScaleSubsSumMinQuotaMap[parQuotaName]; ok { + s.disableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult( + s.disableScaleSubsSumMinQuotaMap[parQuotaName], s.originalMinQuotaMap[subQuotaName]) + } + } + + delete(s.originalMinQuotaMap, subQuotaName) + delete(s.quotaEnableMinQuotaScaleMap, subQuotaName) + klog.V(5).Infof("DeleteScaleMinQuota quota:%v originalMnQuota:%v, enableMinQuotaScale:%v", + subQuotaName, s.originalMinQuotaMap[subQuotaName], s.quotaEnableMinQuotaScaleMap[subQuotaName]) +} + +func (s *ScaleMinQuotaManager) GetScaledMinQuota(newTotalRes v1.ResourceList, parQuotaName, subQuotaName string) (bool, v1.ResourceList) { + s.lock.Lock() + defer s.lock.Unlock() + + if newTotalRes == nil || s.originalMinQuotaMap[subQuotaName] == nil { + return false, nil + } + if s.disableScaleSubsSumMinQuotaMap[parQuotaName] == nil || s.enableScaleSubsSumMinQuotaMap[parQuotaName] == nil { + return false, nil + } + + if !s.quotaEnableMinQuotaScaleMap[subQuotaName] { + return false, nil + } + + // get the dimensions where children's minQuota sum is larger than newTotalRes + needScaleDimensions := make([]v1.ResourceName, 0) + for resName := range newTotalRes { + sum := quotav1.Add(s.disableScaleSubsSumMinQuotaMap[parQuotaName], s.enableScaleSubsSumMinQuotaMap[parQuotaName]) + if newTotalRes.Name(resName, resource.DecimalSI).Cmp(*sum.Name(resName, resource.DecimalSI)) == -1 { + needScaleDimensions = append(needScaleDimensions, resName) + } + } + + // children's minQuota sum is smaller than totalRes in all dimensions + if len(needScaleDimensions) == 0 { + return true, s.originalMinQuotaMap[subQuotaName].DeepCopy() + } + + // ensure the disableScale children's minQuota first + newMinQuota := s.originalMinQuotaMap[subQuotaName].DeepCopy() + for _, resourceDimension := range needScaleDimensions { + needScaleTotal := *newTotalRes.Name(resourceDimension, resource.DecimalSI) + disableTotal := s.disableScaleSubsSumMinQuotaMap[parQuotaName] + needScaleTotal.Sub(*disableTotal.Name(resourceDimension, resource.DecimalSI)) + + if needScaleTotal.Value() <= 0 { + newMinQuota[resourceDimension] = *resource.NewQuantity(0, resource.DecimalSI) + } else { + // if still has minQuota left, enableScaleMinQuota children partition it according to their minQuotaValue. + originalMinQuota := s.originalMinQuotaMap[subQuotaName] + originalMinQuotaValue := originalMinQuota.Name(resourceDimension, resource.DecimalSI) + + enableTotal := s.enableScaleSubsSumMinQuotaMap[parQuotaName] + enableTotalValue := enableTotal.Name(resourceDimension, resource.DecimalSI) + + newMinQuotaValue := int64(0) + if enableTotalValue.Value() > 0 { + newMinQuotaValue = int64(float64(needScaleTotal.Value()) * + float64(originalMinQuotaValue.Value()) / float64(enableTotalValue.Value())) + } + + newMinQuota[resourceDimension] = *resource.NewQuantity(newMinQuotaValue, resource.DecimalSI) + } + } + return true, newMinQuota +} diff --git a/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go new file mode 100644 index 000000000..2518c04d6 --- /dev/null +++ b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go @@ -0,0 +1,304 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + + "github.com/koordinator-sh/koordinator/pkg/util" +) + +func TestScaleMinQuotaWhenOverRootResInfo_GetScaledMinQuota(t *testing.T) { + info := NewScaleMinQuotaManager() + if len(info.enableScaleSubsSumMinQuotaMap) != 0 || + len(info.disableScaleSubsSumMinQuotaMap) != 0 || len(info.originalMinQuotaMap) != 0 || len(info.quotaEnableMinQuotaScaleMap) != 0 { + t.Error("error") + } + { + parQuotaName := "100" + subQuotaName := "1" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := false + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + } + { + parQuotaName := "100" + subQuotaName := "2" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := true + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + } + { + parQuotaName := "100" + subQuotaName := "3" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := true + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + } + + { + totalResource := createResourceList(200, 200) + result, newMinQuota := info.GetScaledMinQuota(totalResource, "101", "1") + if result != false || newMinQuota != nil { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "101", "11") + if result != false || newMinQuota != nil { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "1") + if result != false || newMinQuota != nil { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "2") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(50, 50)) { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(util.NewZeroResourceList(), "100", "2") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(0, 0)) { + t.Error("error") + } + } + + { + totalResource := createResourceList(100, 100) + result, newMinQuota := info.GetScaledMinQuota(totalResource, "100", "1") + if result != false || newMinQuota != nil { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "2") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(25, 25)) { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "3") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(25, 25)) { + t.Error("error") + } + } + { + totalResource := createResourceList(50, 50) + result, newMinQuota := info.GetScaledMinQuota(totalResource, "100", "1") + if result != false || newMinQuota != nil { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "2") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(0, 0)) { + t.Error("error") + } + result, newMinQuota = info.GetScaledMinQuota(totalResource, "100", "3") + if result != true || !quotav1.Equals(newMinQuota, createResourceList(0, 0)) { + t.Error("error") + } + } +} + +func TestScaleMinQuotaWhenOverRootResInfo_UpdateAndDelete(t *testing.T) { + info := NewScaleMinQuotaManager() + if len(info.enableScaleSubsSumMinQuotaMap) != 0 || + len(info.disableScaleSubsSumMinQuotaMap) != 0 || len(info.originalMinQuotaMap) != 0 || len(info.quotaEnableMinQuotaScaleMap) != 0 { + t.Errorf("error") + } + { + parQuotaName := "100" + subQuotaName := "1" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := false + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 1 || + len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 { + t.Errorf("error") + } + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != false { + t.Error("error") + } + } + { + parQuotaName := "100" + subQuotaName := "1" + subMinQuota := createResourceList(40, 40) + enableScaleMinQuota := true + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 1 || + len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 { + t.Errorf("error") + } + + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(40, 40)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(40, 40)) || info.quotaEnableMinQuotaScaleMap["1"] != true { + t.Error("error") + } + } + { + parQuotaName := "100" + subQuotaName := "1" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := true + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 1 || + len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 { + t.Errorf("error") + } + + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true { + t.Error("error") + } + } + { + parQuotaName := "100" + subQuotaName := "2" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := true + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 1 || + len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 2 || len(info.quotaEnableMinQuotaScaleMap) != 2 { + t.Errorf("error") + } + + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(100, 100)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true || + !quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["2"] != true { + t.Error("error") + } + } + { + parQuotaName := "100" + subQuotaName := "3" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := false + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 1 || + len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 3 || len(info.quotaEnableMinQuotaScaleMap) != 3 { + t.Errorf("error") + } + + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(100, 100)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true || + !quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["2"] != true || + !quotav1.Equals(info.originalMinQuotaMap["3"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["3"] != false { + t.Error("error") + } + } + { + parQuotaName := "101" + subQuotaName := "4" + subMinQuota := createResourceList(50, 50) + enableScaleMinQuota := false + info.Update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota) + + if len(info.enableScaleSubsSumMinQuotaMap) != 2 || + len(info.disableScaleSubsSumMinQuotaMap) != 2 || len(info.originalMinQuotaMap) != 4 || len(info.quotaEnableMinQuotaScaleMap) != 4 { + t.Errorf("error") + } + + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(100, 100)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["101"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["101"], createResourceList(50, 50)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true || + !quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["2"] != true || + !quotav1.Equals(info.originalMinQuotaMap["3"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["3"] != false || + !quotav1.Equals(info.originalMinQuotaMap["4"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["4"] != false { + t.Error("error") + } + } + + //begin delete + { + parQuotaName := "101" + subQuotaName := "4" + info.Delete(parQuotaName, subQuotaName) + if len(info.enableScaleSubsSumMinQuotaMap) != 2 || + len(info.disableScaleSubsSumMinQuotaMap) != 2 || len(info.originalMinQuotaMap) != 3 || len(info.quotaEnableMinQuotaScaleMap) != 3 { + t.Errorf("error") + } + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(100, 100)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["101"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["101"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true || + !quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["2"] != true || + !quotav1.Equals(info.originalMinQuotaMap["3"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["3"] != false { + t.Error("error") + } + } + //begin delete + { + parQuotaName := "100" + subQuotaName := "3" + info.Delete(parQuotaName, subQuotaName) + if len(info.enableScaleSubsSumMinQuotaMap) != 2 || + len(info.disableScaleSubsSumMinQuotaMap) != 2 || len(info.originalMinQuotaMap) != 2 || len(info.quotaEnableMinQuotaScaleMap) != 2 { + t.Errorf("error") + } + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(100, 100)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["101"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["101"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true || + !quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["2"] != true { + t.Error("error") + } + } + //begin delete + { + parQuotaName := "100" + subQuotaName := "2" + info.Delete(parQuotaName, subQuotaName) + if len(info.enableScaleSubsSumMinQuotaMap) != 2 || + len(info.disableScaleSubsSumMinQuotaMap) != 2 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 { + t.Errorf("error") + } + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["101"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["101"], createResourceList(0, 0)) || + !quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != true { + t.Error("error") + } + } + //begin delete + { + parQuotaName := "100" + subQuotaName := "1" + info.Delete(parQuotaName, subQuotaName) + if len(info.enableScaleSubsSumMinQuotaMap) != 2 || + len(info.disableScaleSubsSumMinQuotaMap) != 2 || len(info.originalMinQuotaMap) != 0 || len(info.quotaEnableMinQuotaScaleMap) != 0 { + t.Errorf("error") + } + if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) || + !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["101"], v1.ResourceList{}) || + !quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["101"], createResourceList(0, 0)) { + t.Error("error") + } + } +}