scheduler : supports disable runtime quota (koordinator-sh#1839)

Signed-off-by: xulinfei.xlf <xulinfei.xlf@alibaba-inc.com>
Co-authored-by: xulinfei.xlf <xulinfei.xlf@alibaba-inc.com>

pkg/scheduler/apis/config/types.go

@@ -186,6 +186,9 @@ type ElasticQuotaArgs struct {
 
 	// EnableCheckParentQuota check parentQuotaGroups' used and runtime Quota in PreFilter
 	EnableCheckParentQuota *bool
+
+	// EnableRuntimeQuota if true, use max instead of runtime for all checks.
+	EnableRuntimeQuota bool
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/scheduler/apis/config/v1beta2/defaults.go

@@ -67,6 +67,7 @@ var (
 
 	defaultMonitorAllQuotas       = pointer.Bool(false)
 	defaultEnableCheckParentQuota = pointer.Bool(false)
+	defaultEnableRuntimeQuota     = pointer.Bool(true)
 
 	defaultTimeout           = 600 * time.Second
 	defaultControllerWorkers = 1
@@ -167,6 +168,9 @@ func SetDefaults_ElasticQuotaArgs(obj *ElasticQuotaArgs) {
 	if obj.EnableCheckParentQuota == nil {
 		obj.EnableCheckParentQuota = defaultEnableCheckParentQuota
 	}
+	if obj.EnableRuntimeQuota == nil {
+		obj.EnableRuntimeQuota = defaultEnableRuntimeQuota
+	}
 }
 
 func SetDefaults_CoschedulingArgs(obj *CoschedulingArgs) {

pkg/scheduler/apis/config/v1beta2/types.go

@@ -181,6 +181,9 @@ type ElasticQuotaArgs struct {
 
 	// EnableCheckParentQuota check parentQuotaGroups' used and runtime Quota in PreFilter
 	EnableCheckParentQuota *bool `json:"enableCheckParentQuota,omitempty"`
+
+	// EnableRuntimeQuota if false, use max instead of runtime for all checks.
+	EnableRuntimeQuota *bool
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/scheduler/plugins/elasticquota/plugin.go

@@ -60,15 +60,15 @@ type PostFilterState struct {
 	quotaInfo          *core.QuotaInfo
 	used               corev1.ResourceList
 	nonPreemptibleUsed corev1.ResourceList
-	runtime            corev1.ResourceList
+	usedLimit          corev1.ResourceList
 }
 
 func (p *PostFilterState) Clone() framework.StateData {
 	return &PostFilterState{
 		quotaInfo:          p.quotaInfo,
 		used:               p.used.DeepCopy(),
 		nonPreemptibleUsed: p.nonPreemptibleUsed.DeepCopy(),
-		runtime:            p.runtime.DeepCopy(),
+		usedLimit:          p.usedLimit.DeepCopy(),
 	}
 }
 
@@ -229,10 +229,10 @@ func (g *Plugin) PreFilter(ctx context.Context, cycleState *framework.CycleState
 	podRequest, _ := core.PodRequestsAndLimits(pod)
 
 	used := quotav1.Mask(quotav1.Add(podRequest, state.used), quotav1.ResourceNames(podRequest))
-	if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(used, state.runtime); !isLessEqual {
+	if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(used, state.usedLimit); !isLessEqual {
 		return nil, framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient quotas, "+
 			"quotaName: %v, runtime: %v, used: %v, pod's request: %v, exceedDimensions: %v",
-			quotaName, printResourceList(state.runtime), printResourceList(state.used), printResourceList(podRequest), exceedDimensions))
+			quotaName, printResourceList(state.usedLimit), printResourceList(state.used), printResourceList(podRequest), exceedDimensions))
 	}
 
 	if extension.IsPodNonPreemptible(pod) {

pkg/scheduler/plugins/elasticquota/plugin_helper.go

@@ -18,7 +18,6 @@ package elasticquota
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sort"
 	"strings"
@@ -171,8 +170,6 @@ func (g *Plugin) createDefaultQuotaIfNotPresent() {
 			Max: g.pluginArgs.DefaultQuotaGroupMax.DeepCopy(),
 		},
 	}
-	sharedWeight, _ := json.Marshal(defaultElasticQuota.Spec.Max)
-	defaultElasticQuota.Annotations[extension.AnnotationRuntime] = string(sharedWeight)
 	_, err := g.client.SchedulingV1alpha1().ElasticQuotas(g.pluginArgs.QuotaGroupNamespace).
 		Create(context.TODO(), defaultElasticQuota, metav1.CreateOptions{})
 	if err != nil {
@@ -201,8 +198,6 @@ func (g *Plugin) createSystemQuotaIfNotPresent() {
 			Max: g.pluginArgs.SystemQuotaGroupMax.DeepCopy(),
 		},
 	}
-	sharedWeight, _ := json.Marshal(systemElasticQuota.Spec.Max)
-	systemElasticQuota.Annotations[extension.AnnotationRuntime] = string(sharedWeight)
 	_, err := g.client.SchedulingV1alpha1().ElasticQuotas(g.pluginArgs.QuotaGroupNamespace).
 		Create(context.TODO(), systemElasticQuota, metav1.CreateOptions{})
 	if err != nil {
@@ -244,7 +239,7 @@ func (g *Plugin) snapshotPostFilterState(quotaInfo *core.QuotaInfo, state *frame
 		quotaInfo:          quotaInfo,
 		used:               quotaInfo.GetUsed(),
 		nonPreemptibleUsed: quotaInfo.GetNonPreemptibleUsed(),
-		runtime:            quotaInfo.GetRuntime(),
+		usedLimit:          g.getQuotaInfoUsedLimit(quotaInfo),
 	}
 	state.Write(postFilterKey, postFilterState)
 	return postFilterState
@@ -273,12 +268,13 @@ func getPostFilterState(cycleState *framework.CycleState) (*PostFilterState, err
 func (g *Plugin) checkQuotaRecursive(curQuotaName string, quotaNameTopo []string, podRequest v1.ResourceList) *framework.Status {
 	quotaInfo := g.groupQuotaManager.GetQuotaInfoByName(curQuotaName)
 	quotaUsed := quotaInfo.GetUsed()
-	quotaRuntime := quotaInfo.GetRuntime()
+	quotaUsedLimit := g.getQuotaInfoUsedLimit(quotaInfo)
+
 	newUsed := quotav1.Mask(quotav1.Add(podRequest, quotaUsed), quotav1.ResourceNames(podRequest))
-	if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(newUsed, quotaRuntime); !isLessEqual {
+	if isLessEqual, exceedDimensions := quotav1.LessThanOrEqual(newUsed, quotaUsedLimit); !isLessEqual {
 		return framework.NewStatus(framework.Unschedulable, fmt.Sprintf("Insufficient quotas, "+
 			"quotaNameTopo: %v, runtime: %v, used: %v, pod's request: %v, exceedDimensions: %v", quotaNameTopo,
-			printResourceList(quotaRuntime), printResourceList(quotaUsed), printResourceList(podRequest), exceedDimensions))
+			printResourceList(quotaUsedLimit), printResourceList(quotaUsed), printResourceList(podRequest), exceedDimensions))
 	}
 	if quotaInfo.ParentName != extension.RootQuotaName {
 		quotaNameTopo = append([]string{quotaInfo.ParentName}, quotaNameTopo...)
@@ -298,3 +294,10 @@ func printResourceList(rl v1.ResourceList) string {
 	})
 	return strings.Join(res, ",")
 }
+
+func (g *Plugin) getQuotaInfoUsedLimit(quotaInfo *core.QuotaInfo) v1.ResourceList {
+	if g.pluginArgs.EnableRuntimeQuota {
+		return quotaInfo.GetRuntime()
+	}
+	return quotaInfo.GetMax()
+}

pkg/scheduler/plugins/elasticquota/plugin_helper_test.go

@@ -27,6 +27,8 @@ import (
 	schedulerv1alpha1 "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1"
 
 	"github.com/koordinator-sh/koordinator/apis/extension"
+	"github.com/koordinator-sh/koordinator/pkg/scheduler/apis/config"
+	"github.com/koordinator-sh/koordinator/pkg/scheduler/plugins/elasticquota/core"
 )
 
 func TestGetQuotaName(t *testing.T) {
@@ -105,3 +107,52 @@ func TestGetQuotaName(t *testing.T) {
 		})
 	}
 }
+
+func TestPlugin_getQuotaInfoRuntime(t *testing.T) {
+	type args struct {
+		quotaInfo          *core.QuotaInfo
+		enableRuntimeQuota bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want corev1.ResourceList
+	}{
+		{
+			name: "get runtime",
+			args: args{
+				enableRuntimeQuota: true,
+				quotaInfo: &core.QuotaInfo{
+					CalculateInfo: core.QuotaCalculateInfo{
+						Max:     createResourceList(100, 100),
+						Runtime: createResourceList(1, 1),
+					},
+				},
+			},
+			want: createResourceList(1, 1),
+		},
+		{
+			name: "get max",
+			args: args{
+				enableRuntimeQuota: false,
+				quotaInfo: &core.QuotaInfo{
+					CalculateInfo: core.QuotaCalculateInfo{
+						Max:     createResourceList(100, 100),
+						Runtime: createResourceList(1, 1),
+					},
+				},
+			},
+			want: createResourceList(100, 100),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := &Plugin{
+				pluginArgs: &config.ElasticQuotaArgs{
+					EnableRuntimeQuota: tt.args.enableRuntimeQuota,
+				},
+			}
+			assert.Equalf(t, tt.want, g.getQuotaInfoUsedLimit(tt.args.quotaInfo), "getQuotaInfoUsedLimit(%v)", tt.args.quotaInfo)
+		})
+	}
+}

pkg/scheduler/plugins/elasticquota/plugin_test.go

@@ -602,11 +602,12 @@ func setLoglevel(logLevel string) {
 
 func TestPlugin_PreFilter(t *testing.T) {
 	test := []struct {
-		name           string
-		pod            *corev1.Pod
-		quotaInfo      *core.QuotaInfo
-		expectedStatus *framework.Status
-		checkParent    bool
+		name                string
+		pod                 *corev1.Pod
+		quotaInfo           *core.QuotaInfo
+		expectedStatus      *framework.Status
+		checkParent         bool
+		disableRuntimeQuota bool
 	}{
 		{
 			name: "default",
@@ -663,13 +664,28 @@ func TestPlugin_PreFilter(t *testing.T) {
 			},
 			expectedStatus: framework.NewStatus(framework.Success, ""),
 		},
+		{
+			name: "runtime not enough, but disable runtime",
+			pod: MakePod("t1-ns1", "pod1").Container(
+				MakeResourceList().CPU(1).Mem(3).GPU(1).Obj()).Obj(),
+			quotaInfo: &core.QuotaInfo{
+				Name: extension.DefaultQuotaName,
+				CalculateInfo: core.QuotaCalculateInfo{
+					Max:     MakeResourceList().CPU(1).Mem(3).Obj(),
+					Runtime: MakeResourceList().CPU(1).Mem(2).Obj(),
+				},
+			},
+			disableRuntimeQuota: true,
+			expectedStatus:      framework.NewStatus(framework.Success, ""),
+		},
 	}
 	for _, tt := range test {
 		t.Run(tt.name, func(t *testing.T) {
 			suit := newPluginTestSuit(t, nil)
 			p, err := suit.proxyNew(suit.elasticQuotaArgs, suit.Handle)
 			assert.Nil(t, err)
 			gp := p.(*Plugin)
+			gp.pluginArgs.EnableRuntimeQuota = !tt.disableRuntimeQuota
 			qi := gp.groupQuotaManager.GetQuotaInfoByName(tt.quotaInfo.Name)
 			qi.Lock()
 			qi.CalculateInfo.Runtime = tt.quotaInfo.CalculateInfo.Runtime.DeepCopy()

pkg/scheduler/plugins/elasticquota/preempt.go

@@ -188,7 +188,7 @@ func (g *Plugin) SelectVictimsOnNode(
 		}
 
 		newUsed := quotav1.Mask(quotav1.Add(postFilterState.used, podReq), quotav1.ResourceNames(podReq))
-		if isLessEqual, _ := quotav1.LessThanOrEqual(newUsed, postFilterState.runtime); !isLessEqual {
+		if isLessEqual, _ := quotav1.LessThanOrEqual(newUsed, postFilterState.usedLimit); !isLessEqual {
 			if err := removePod(pi); err != nil {
 				return false, err
 			}

pkg/scheduler/plugins/elasticquota/quota_overuse_revoke.go

@@ -152,6 +152,7 @@ type QuotaOverUsedRevokeController struct {
 	overUsedTriggerEvictDuration time.Duration
 	revokePodCycle               time.Duration
 	monitorAllQuotas             bool
+	enableRuntimeQuota           bool
 	plugin                       *Plugin
 }
 
@@ -165,6 +166,8 @@ func NewQuotaOverUsedRevokeController(plugin *Plugin) *QuotaOverUsedRevokeContro
 	if plugin.pluginArgs.MonitorAllQuotas != nil {
 		controller.monitorAllQuotas = *plugin.pluginArgs.MonitorAllQuotas
 	}
+	controller.enableRuntimeQuota = plugin.pluginArgs.EnableRuntimeQuota
+
 	return controller
 }
 
@@ -177,6 +180,10 @@ func (controller *QuotaOverUsedRevokeController) Start() {
 		klog.Infof("monitorAllQuotas not true. will not start elasticQuota QuotaOverUsedRevokeController")
 		return
 	}
+	if !controller.enableRuntimeQuota {
+		klog.Infof("enableRuntimeQuota is false. will not start elasticQuota QuotaOverUsedRevokeController")
+		return
+	}
 	go wait.Until(controller.revokePodDueToQuotaOverUsed, controller.revokePodCycle, nil)
 	klog.Infof("start elasticQuota QuotaOverUsedRevokeController")
 }