From 65339da5dab62760964ade2a7687b283b349ff77 Mon Sep 17 00:00:00 2001 From: shaloulcy Date: Wed, 28 Feb 2024 10:04:37 +0800 Subject: [PATCH] scheduler: add unschedulable resource for root quota (#1925) Signed-off-by: chuanyun.lcy Co-authored-by: chuanyun.lcy --- apis/extension/elastic_quota.go | 11 +++ .../profile/profile_controller.go | 14 +++ .../profile/profile_controller_test.go | 90 +++++++++++++------ .../plugins/elasticquota/controller.go | 9 ++ .../plugins/elasticquota/controller_test.go | 10 +++ 5 files changed, 107 insertions(+), 27 deletions(-) diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go index b061eb8ee..16e76ff12 100644 --- a/apis/extension/elastic_quota.go +++ b/apis/extension/elastic_quota.go @@ -47,6 +47,7 @@ const ( AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request" AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys" AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource" + AnnotationUnschedulableResource = QuotaKoordinatorPrefix + "/unschedulable-resource" AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces" AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed" AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated" @@ -195,3 +196,13 @@ func GetChildRequest(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) } return request, nil } + +func GetUnschedulableResource(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) { + unschedulable := corev1.ResourceList{} + if quota.Annotations[AnnotationUnschedulableResource] != "" { + if err := json.Unmarshal([]byte(quota.Annotations[AnnotationUnschedulableResource]), &unschedulable); err != nil { + return unschedulable, err + } + } + return unschedulable, nil +} diff --git a/pkg/quota-controller/profile/profile_controller.go b/pkg/quota-controller/profile/profile_controller.go index 3aea4f394..b3d6ac698 100644 --- a/pkg/quota-controller/profile/profile_controller.go +++ b/pkg/quota-controller/profile/profile_controller.go @@ -35,6 +35,7 @@ import ( quotav1 "k8s.io/apiserver/pkg/quota/v1" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" + nodeutil "k8s.io/kubernetes/pkg/util/node" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -138,11 +139,16 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request // TODO: consider node status. totalResource := corev1.ResourceList{} + unschedulableResource := corev1.ResourceList{} for _, node := range nodeList.Items { totalResource = quotav1.Add(totalResource, GetNodeAllocatable(node)) + if node.Spec.Unschedulable || !nodeutil.IsNodeReady(&node) { + unschedulableResource = quotav1.Add(unschedulableResource, GetNodeAllocatable(node)) + } } decorateTotalResource(profile, totalResource) + decorateTotalResource(profile, unschedulableResource) resourceKeys := []string{"cpu", "memory"} raw, ok := profile.Annotations[extension.AnnotationResourceKeys] @@ -197,6 +203,14 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request } quota.Annotations[extension.AnnotationTotalResource] = string(data) + // update unschedulable resource + data, err = json.Marshal(unschedulableResource) + if err != nil { + klog.Errorf("failed marshal unschedulable resources, err: %v", err) + return ctrl.Result{Requeue: true}, err + } + quota.Annotations[extension.AnnotationUnschedulableResource] = string(data) + if !quotaExist { err = r.Client.Create(context.TODO(), quota) if err != nil { diff --git a/pkg/quota-controller/profile/profile_controller_test.go b/pkg/quota-controller/profile/profile_controller_test.go index b0987622d..f1013cf78 100644 --- a/pkg/quota-controller/profile/profile_controller_test.go +++ b/pkg/quota-controller/profile/profile_controller_test.go @@ -65,6 +65,30 @@ func defaultCreateNode(nodeName string, labels map[string]string, capacity corev }, Status: corev1.NodeStatus{ Allocatable: capacity, + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + } +} + +func defaultCreateUnreadyNode(nodeName string, labels map[string]string, capacity corev1.ResourceList) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: labels, + }, + Status: corev1.NodeStatus{ + Allocatable: capacity, + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, }, } } @@ -77,7 +101,7 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { nodes := []*corev1.Node{ defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)), - defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)), + defaultCreateUnreadyNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)), defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceListWithStorage(10, 1000, 1000)), } @@ -87,12 +111,13 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { resourceRatio := "0.9" tests := []struct { - name string - profile *quotav1alpha1.ElasticQuotaProfile - oriQuota *schedv1alpha1.ElasticQuota - expectQuotaMin corev1.ResourceList - expectTotalResource corev1.ResourceList - expectQuotaLabels map[string]string + name string + profile *quotav1alpha1.ElasticQuotaProfile + oriQuota *schedv1alpha1.ElasticQuota + expectQuotaMin corev1.ResourceList + expectTotalResource corev1.ResourceList + expectUnschedulableResource corev1.ResourceList + expectQuotaLabels map[string]string }{ { name: "cn-hangzhou-a profile", @@ -107,9 +132,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: createResourceList(20, 2000), - expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + oriQuota: nil, + expectQuotaMin: createResourceList(20, 2000), + expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: treeID1, @@ -129,9 +155,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: createResourceList(10, 1000), - expectTotalResource: createResourceListWithStorage(10, 1000, 1000), + oriQuota: nil, + expectQuotaMin: createResourceList(10, 1000), + expectTotalResource: createResourceListWithStorage(10, 1000, 1000), + expectUnschedulableResource: corev1.ResourceList{}, expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile2", extension.LabelQuotaTreeID: treeID2, @@ -154,9 +181,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: createResourceList(20, 2000), - expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + oriQuota: nil, + expectQuotaMin: createResourceList(20, 2000), + expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: treeID1, @@ -189,8 +217,9 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { Min: createResourceList(5, 50), }, }, - expectQuotaMin: createResourceList(20, 2000), - expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + expectQuotaMin: createResourceList(20, 2000), + expectTotalResource: createResourceListWithStorage(20, 2000, 2000), + expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: treeID1, @@ -213,9 +242,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: createResourceList(18, 1800), - expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + oriQuota: nil, + expectQuotaMin: createResourceList(18, 1800), + expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + expectUnschedulableResource: createResourceListWithStorage(9, 900, 900), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: treeID1, @@ -239,9 +269,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: createResourceList(18, 1800), - expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + oriQuota: nil, + expectQuotaMin: createResourceList(18, 1800), + expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + expectUnschedulableResource: createResourceListWithStorage(9, 900, 900), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: "tree1", @@ -268,9 +299,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { }, }, }, - oriQuota: nil, - expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)}, - expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + oriQuota: nil, + expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)}, + expectTotalResource: createResourceListWithStorage(18, 1800, 1800), + expectUnschedulableResource: createResourceListWithStorage(9, 900, 900), expectQuotaLabels: map[string]string{ extension.LabelQuotaProfile: "profile1", extension.LabelQuotaTreeID: "tree1", @@ -310,8 +342,12 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) { err = json.Unmarshal([]byte(quota.Annotations[extension.AnnotationTotalResource]), &total) assert.NoError(t, err) + unschedulable, err := extension.GetUnschedulableResource(quota) + assert.NoError(t, err) + assert.True(t, quotav1.Equals(tc.expectQuotaMin, quota.Spec.Min)) assert.True(t, quotav1.Equals(tc.expectTotalResource, total)) + assert.True(t, quotav1.Equals(tc.expectUnschedulableResource, unschedulable)) assert.Equal(t, tc.expectQuotaLabels, quota.Labels) }) } diff --git a/pkg/scheduler/plugins/elasticquota/controller.go b/pkg/scheduler/plugins/elasticquota/controller.go index 29c1215f6..d30b9f765 100644 --- a/pkg/scheduler/plugins/elasticquota/controller.go +++ b/pkg/scheduler/plugins/elasticquota/controller.go @@ -257,6 +257,15 @@ func syncElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoS extension.AnnotationNonPreemptibleRequest: summary.NonPreemptibleRequest, extension.AnnotationNonPreemptibleUsed: summary.NonPreemptibleUsed, } + + // record the unschedulable resource + if extension.IsTreeRootQuota(eq) { + unschedulable, err := extension.GetUnschedulableResource(eq) + if err == nil { + m[extension.AnnotationUnschedulableResource] = unschedulable + } + } + for k, v := range m { decorateResource(eq, v) diff --git a/pkg/scheduler/plugins/elasticquota/controller_test.go b/pkg/scheduler/plugins/elasticquota/controller_test.go index a65f8a196..d63b1690c 100644 --- a/pkg/scheduler/plugins/elasticquota/controller_test.go +++ b/pkg/scheduler/plugins/elasticquota/controller_test.go @@ -367,6 +367,14 @@ func Test_updateElasticQuotaStatusIfChanged(t *testing.T) { func Test_syncElasticQuotaMetrics(t *testing.T) { eq := &v1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "quota.scheduling.koordinator.sh/is-root": "true", + }, + Annotations: map[string]string{ + "quota.scheduling.koordinator.sh/unschedulable-resource": `{"cpu":"4","memory":"8"}`, + }, + }, Spec: v1alpha1.ElasticQuotaSpec{ Max: MakeResourceList().CPU(4).Mem(200).Obj(), Min: MakeResourceList().CPU(2).Mem(100).Obj(), @@ -435,6 +443,8 @@ loopChan: `label: label: label: label: label: label: gauge: `, `label: label: label: label: label: label: gauge: `, `label: label: label: label: label: label: gauge: `, + `label: label: label: label: label: label: gauge: `, + `label: label: label: label: label: label: gauge: `, `label: label: label: label: label: label: gauge: `, `label: label: label: label: label: label: gauge: `, }