Skip to content

Commit

Permalink
webhook: add quota evaluate webhook (#2129)
Browse files Browse the repository at this point in the history 
Signed-off-by: chuanyun.lcy <chuanyun.lcy@alibaba-inc.com>
Co-authored-by: chuanyun.lcy <chuanyun.lcy@alibaba-inc.com>
  • Loading branch information
@shaloulcy
shaloulcy and chuanyun.lcy committed Jul 10, 2024
1 parent 448015c commit 6861c87
Show file tree
Hide file tree
Showing 16 changed files with 1,486 additions and 157 deletions.
11 changes: 11 additions & 0 deletions apis/extension/elastic_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ const (
AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated"
AnnotationNonPreemptibleRequest = QuotaKoordinatorPrefix + "/non-preemptible-request"
AnnotationNonPreemptibleUsed = QuotaKoordinatorPrefix + "/non-preemptible-used"
AnnotationAdmission = QuotaKoordinatorPrefix + "/admission"
)

func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string {
Expand Down Expand Up @@ -207,3 +208,13 @@ func GetUnschedulableResource(quota *v1alpha1.ElasticQuota) (corev1.ResourceList
}
return unschedulable, nil
}

func GetAdmission(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
admission := corev1.ResourceList{}
if quota.Annotations[AnnotationAdmission] != "" {
if err := json.Unmarshal([]byte(quota.Annotations[AnnotationAdmission]), &admission); err != nil {
return admission, err
}
}
return admission, nil
}
4 changes: 4 additions & 0 deletions pkg/features/features.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ const (

// SupportParentQuotaSubmitPod enables parent Quota submit pod
SupportParentQuotaSubmitPod featuregate.Feature = "SupportParentQuotaSubmitPod"

// EnableQuotaAdmission enables quota admission.
EnableQuotaAdmission featuregate.Feature = "EnableQuotaAdmission"
)

var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
Expand All @@ -84,6 +87,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
ElasticQuotaGuaranteeUsage: {Default: false, PreRelease: featuregate.Alpha},
DisableDefaultQuota: {Default: false, PreRelease: featuregate.Alpha},
SupportParentQuotaSubmitPod: {Default: false, PreRelease: featuregate.Alpha},
EnableQuotaAdmission: {Default: false, PreRelease: featuregate.Alpha},
}

const (
Expand Down
15 changes: 11 additions & 4 deletions pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,11 @@ func (gqm *GroupQuotaManager) GetAllQuotaNames() map[string]struct{} {
}

func (gqm *GroupQuotaManager) updatePodRequestNoLock(quotaName string, oldPod, newPod *v1.Pod) {
quotaInfo := gqm.getQuotaInfoByNameNoLock(quotaName)
if quotaInfo == nil {
return
}

var oldPodReq, newPodReq, oldNonPreemptibleRequest, newNonPreemptibleRequest v1.ResourceList
if oldPod != nil {
oldPodReq = PodRequests(oldPod)
Expand All @@ -609,8 +614,9 @@ func (gqm *GroupQuotaManager) updatePodRequestNoLock(quotaName string, oldPod, n
}
}

deltaReq := quotav1.Subtract(newPodReq, oldPodReq)
deltaNonPreemptibleRequest := quotav1.Subtract(newNonPreemptibleRequest, oldNonPreemptibleRequest)
resourceNames := quotav1.ResourceNames(quotaInfo.CalculateInfo.Max)
deltaReq := quotav1.Mask(quotav1.Subtract(newPodReq, oldPodReq), resourceNames)
deltaNonPreemptibleRequest := quotav1.Mask(quotav1.Subtract(newNonPreemptibleRequest, oldNonPreemptibleRequest), resourceNames)
if quotav1.IsZero(deltaReq) && quotav1.IsZero(deltaNonPreemptibleRequest) {
return
}
Expand Down Expand Up @@ -643,8 +649,9 @@ func (gqm *GroupQuotaManager) updatePodUsedNoLock(quotaName string, oldPod, newP
}
}

deltaUsed := quotav1.Subtract(newPodUsed, oldPodUsed)
deltaNonPreemptibleUsed := quotav1.Subtract(newNonPreemptibleUsed, oldNonPreemptibleUsed)
resourceNames := quotav1.ResourceNames(quotaInfo.CalculateInfo.Max)
deltaUsed := quotav1.Mask(quotav1.Subtract(newPodUsed, oldPodUsed), resourceNames)
deltaNonPreemptibleUsed := quotav1.Mask(quotav1.Subtract(newNonPreemptibleUsed, oldNonPreemptibleUsed), resourceNames)
if quotav1.IsZero(deltaUsed) && quotav1.IsZero(deltaNonPreemptibleUsed) {
if klog.V(5).Enabled() {
klog.Infof("updatePodUsed, deltaUsedIsZero and deltaNonPreemptibleUsedIsZero, quotaName: %v, podName: %v, podUsed: %v, podNonPreemptibleUsed: %v",
Expand Down
15 changes: 13 additions & 2 deletions pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package core
import (
"encoding/json"
"fmt"
"math"
"math/rand"
"testing"
"time"
Expand Down Expand Up @@ -1316,8 +1317,18 @@ func NewGroupQuotaManagerForTest() *GroupQuotaManager {
scaleMinQuotaManager: NewScaleMinQuotaManager(),
quotaTopoNodeMap: make(map[string]*QuotaTopoNode),
}
quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, extension.RootQuotaName)
quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName)
systemQuotaInfo := NewQuotaInfo(false, true, extension.SystemQuotaName, extension.RootQuotaName)
systemQuotaInfo.CalculateInfo.Max = v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(math.MaxInt64/5, resource.BinarySI),
}
defaultQuotaInfo := NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName)
defaultQuotaInfo.CalculateInfo.Max = v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(math.MaxInt64/5, resource.BinarySI),
}
quotaManager.quotaInfoMap[extension.SystemQuotaName] = systemQuotaInfo
quotaManager.quotaInfoMap[extension.DefaultQuotaName] = defaultQuotaInfo
quotaManager.quotaInfoMap[extension.RootQuotaName] = NewQuotaInfo(true, false, extension.RootQuotaName, "")
quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName)
return quotaManager
Expand Down
7 changes: 5 additions & 2 deletions pkg/scheduler/plugins/elasticquota/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,8 @@ func (g *Plugin) PreFilter(ctx context.Context, cycleState *framework.CycleState
state := g.snapshotPostFilterState(quotaInfo, cycleState)

podRequest := core.PodRequests(pod)
used := quotav1.Mask(quotav1.Add(podRequest, state.used), quotav1.ResourceNames(podRequest))
podRequest = quotav1.Mask(podRequest, quotav1.ResourceNames(quotaInfo.CalculateInfo.Max))
used := quotav1.Add(podRequest, state.used)
if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(used, state.usedLimit); !isLessEqual {
return nil, framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient quotas, "+
"quotaName: %v, runtime: %v, used: %v, pod's request: %v, exceedDimensions: %v",
Expand All @@ -238,7 +239,7 @@ func (g *Plugin) PreFilter(ctx context.Context, cycleState *framework.CycleState
if extension.IsPodNonPreemptible(pod) {
quotaMin := state.quotaInfo.CalculateInfo.Min
nonPreemptibleUsed := state.nonPreemptibleUsed
addNonPreemptibleUsed := quotav1.Mask(quotav1.Add(podRequest, nonPreemptibleUsed), quotav1.ResourceNames(podRequest))
addNonPreemptibleUsed := quotav1.Add(podRequest, nonPreemptibleUsed)
if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(addNonPreemptibleUsed, quotaMin); !isLessEqual {
return nil, framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient non-preemptible quotas, "+
"quotaName: %v, min: %v, nonPreemptibleUsed: %v, pod's request: %v, exceedDimensions: %v",
Expand Down Expand Up @@ -272,6 +273,7 @@ func (g *Plugin) AddPod(ctx context.Context, state *framework.CycleState, podToS

if postFilterState.quotaInfo.IsPodExist(podInfoToAdd.Pod) {
podReq := core.PodRequests(podInfoToAdd.Pod)
podReq = quotav1.Mask(podReq, quotav1.ResourceNames(postFilterState.quotaInfo.CalculateInfo.Max))
postFilterState.used = quotav1.Add(postFilterState.used, podReq)
}
return framework.NewStatus(framework.Success, "")
Expand All @@ -292,6 +294,7 @@ func (g *Plugin) RemovePod(ctx context.Context, state *framework.CycleState, pod

if postFilterState.quotaInfo.IsPodExist(podInfoToRemove.Pod) {
podReq := core.PodRequests(podInfoToRemove.Pod)
podReq = quotav1.Mask(podReq, quotav1.ResourceNames(postFilterState.quotaInfo.CalculateInfo.Max))
postFilterState.used = quotav1.SubtractWithNonNegativeResult(postFilterState.used, podReq)
}
return framework.NewStatus(framework.Success, "")
Expand Down
26 changes: 13 additions & 13 deletions pkg/scheduler/plugins/elasticquota/plugin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,13 +626,13 @@ func TestPlugin_PreFilter(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Runtime: MakeResourceList().CPU(0).Mem(20).GPU(10).Obj(),
Runtime: MakeResourceList().CPU(0).Mem(20).Obj(),
},
},
expectedStatus: framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient quotas, "+
"quotaName: %v, runtime: %v, used: %v, pod's request: %v, exceedDimensions: [cpu]",
extension.DefaultQuotaName, printResourceList(MakeResourceList().CPU(0).Mem(20).GPU(10).Obj()),
printResourceList(corev1.ResourceList{}), printResourceList(MakeResourceList().CPU(1).Mem(2).GPU(1).Obj()))),
extension.DefaultQuotaName, printResourceList(MakeResourceList().CPU(0).Mem(20).Obj()),
printResourceList(corev1.ResourceList{}), printResourceList(MakeResourceList().CPU(1).Mem(2).Obj()))),
},
{
name: "used dimension larger than runtime, but value is enough",
Expand All @@ -641,7 +641,7 @@ func TestPlugin_PreFilter(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Runtime: MakeResourceList().CPU(10).Mem(20).GPU(10).Obj(),
Runtime: MakeResourceList().CPU(10).Mem(20).Obj(),
},
},
expectedStatus: framework.NewStatus(framework.Success, ""),
Expand All @@ -653,14 +653,15 @@ func TestPlugin_PreFilter(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Max: MakeResourceList().CPU(10).Mem(20).Obj(),
Runtime: MakeResourceList().CPU(1).Mem(2).Obj(),
},
},
expectedStatus: framework.NewStatus(framework.Unschedulable,
fmt.Sprintf("Insufficient quotas, "+
"quotaName: %v, runtime: %v, used: %v, pod's request: %v, exceedDimensions: [memory]",
extension.DefaultQuotaName, printResourceList(MakeResourceList().CPU(1).Mem(2).Obj()),
printResourceList(corev1.ResourceList{}), printResourceList(MakeResourceList().CPU(1).Mem(3).GPU(1).Obj()))),
printResourceList(corev1.ResourceList{}), printResourceList(MakeResourceList().CPU(1).Mem(3).Obj()))),
},
{
name: "used dimension larger than runtime, but value is enough",
Expand All @@ -669,7 +670,7 @@ func TestPlugin_PreFilter(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Runtime: MakeResourceList().CPU(10).Mem(20).GPU(10).Obj(),
Runtime: MakeResourceList().CPU(10).Mem(20).Obj(),
},
},
expectedStatus: framework.NewStatus(framework.Success, ""),
Expand All @@ -681,7 +682,6 @@ func TestPlugin_PreFilter(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Max: MakeResourceList().CPU(1).Mem(3).Obj(),
Runtime: MakeResourceList().CPU(1).Mem(2).Obj(),
},
},
Expand Down Expand Up @@ -898,10 +898,10 @@ func TestPlugin_Reserve(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Used: MakeResourceList().CPU(10).Mem(20).GPU(10).Obj(),
Used: MakeResourceList().CPU(10).Mem(20).Obj(),
},
},
expectedUsed: MakeResourceList().CPU(11).Mem(22).GPU(11).Obj(),
expectedUsed: MakeResourceList().CPU(11).Mem(22).Obj(),
},
}
for _, tt := range test {
Expand Down Expand Up @@ -977,12 +977,12 @@ func TestPlugin_AddPod(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Used: MakeResourceList().CPU(10).Mem(20).GPU(10).Obj(),
Used: MakeResourceList().CPU(10).Mem(20).Obj(),
},
PodCache: make(map[string]*core.PodInfo),
},
wantStatusSuccess: true,
expectedUsed: MakeResourceList().CPU(11).Mem(22).GPU(11).Obj(),
expectedUsed: MakeResourceList().CPU(11).Mem(22).Obj(),
},
}
for _, tt := range test {
Expand Down Expand Up @@ -1029,11 +1029,11 @@ func TestPlugin_RemovePod(t *testing.T) {
quotaInfo: &core.QuotaInfo{
Name: extension.DefaultQuotaName,
CalculateInfo: core.QuotaCalculateInfo{
Used: MakeResourceList().CPU(10).Mem(20).GPU(10).Obj(),
Used: MakeResourceList().CPU(10).Mem(20).Obj(),
},
},
wantStatusSuccess: true,
expectedUsed: MakeResourceList().CPU(9).Mem(18).GPU(9).Obj(),
expectedUsed: MakeResourceList().CPU(9).Mem(18).Obj(),
},
}
for _, tt := range test {
Expand Down
129 changes: 0 additions & 129 deletions pkg/webhook/elasticquota/quota_topology_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ import (

"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
testing2 "k8s.io/kubernetes/pkg/scheduler/testing"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

Expand Down Expand Up @@ -925,130 +923,3 @@ func TestQuotaTopology_checkGuaranteeForMin(t *testing.T) {
})
}
}

type podWrapper struct{ *v1.Pod }

func MakePod(namespace, name string) *podWrapper {
pod := testing2.MakePod().Namespace(namespace).Name(name).Obj()

return &podWrapper{pod}
}

func (p *podWrapper) Label(string1, string2 string) *podWrapper {
if p.Labels == nil {
p.Labels = make(map[string]string)
}
p.Labels[string1] = string2
return p
}

func (p *podWrapper) Obj() *v1.Pod {
return p.Pod
}

type quotaWrapper struct {
*v1alpha1.ElasticQuota
}

func MakeQuota(name string) *quotaWrapper {
eq := &v1alpha1.ElasticQuota{
TypeMeta: metav1.TypeMeta{Kind: "ElasticQuota", APIVersion: "scheduling.sigs.k8s.io/v1alpha1"},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: make(map[string]string),
Annotations: make(map[string]string),
},
}
return &quotaWrapper{eq}
}

func (q *quotaWrapper) Namespace(ns string) *quotaWrapper {
q.ElasticQuota.Namespace = ns
return q
}

func (q *quotaWrapper) Min(min v1.ResourceList) *quotaWrapper {
q.ElasticQuota.Spec.Min = min
return q
}

func (q *quotaWrapper) Max(max v1.ResourceList) *quotaWrapper {
q.ElasticQuota.Spec.Max = max
return q
}

func (q *quotaWrapper) TreeID(tree string) *quotaWrapper {
q.ElasticQuota.Labels[extension.LabelQuotaTreeID] = tree
return q
}

func (q *quotaWrapper) Guaranteed(guaranteed v1.ResourceList) *quotaWrapper {
raw, err := json.Marshal(guaranteed)
if err == nil {
q.ElasticQuota.Annotations[extension.AnnotationGuaranteed] = string(raw)
}
return q
}

func (q *quotaWrapper) IsRoot(isRoot bool) *quotaWrapper {
if isRoot {
q.Labels[extension.LabelQuotaIsRoot] = "true"
}
return q
}

func (q *quotaWrapper) sharedWeight(sharedWeight v1.ResourceList) *quotaWrapper {
sharedWeightBytes, _ := json.Marshal(sharedWeight)
q.ElasticQuota.Annotations[extension.AnnotationSharedWeight] = string(sharedWeightBytes)
return q
}

func (q *quotaWrapper) IsParent(isParent bool) *quotaWrapper {
if isParent {
q.Labels[extension.LabelQuotaIsParent] = "true"
} else {
q.Labels[extension.LabelQuotaIsParent] = "false"
}
return q
}

func (q *quotaWrapper) ParentName(parentName string) *quotaWrapper {
q.Labels[extension.LabelQuotaParent] = parentName
return q
}

func (q *quotaWrapper) Annotations(annotations map[string]string) *quotaWrapper {
for k, v := range annotations {
q.ElasticQuota.Annotations[k] = v
}
return q
}

func (q *quotaWrapper) Obj() *v1alpha1.ElasticQuota {
return q.ElasticQuota
}

type resourceWrapper struct{ v1.ResourceList }

func MakeResourceList() *resourceWrapper {
return &resourceWrapper{v1.ResourceList{}}
}

func (r *resourceWrapper) CPU(val int64) *resourceWrapper {
r.ResourceList[v1.ResourceCPU] = *resource.NewQuantity(val, resource.DecimalSI)
return r
}

func (r *resourceWrapper) Mem(val int64) *resourceWrapper {
r.ResourceList[v1.ResourceMemory] = *resource.NewQuantity(val, resource.DecimalSI)
return r
}

func (r *resourceWrapper) GPU(val int64) *resourceWrapper {
r.ResourceList["nvidia.com/gpu"] = *resource.NewQuantity(val, resource.DecimalSI)
return r
}

func (r *resourceWrapper) Obj() v1.ResourceList {
return r.ResourceList
}

0 comments on commit 6861c87

Please sign in to comment.