Skip to content

Commit

Permalink
scheduler: add unschedulable resource for root quota (#1925)
Browse files Browse the repository at this point in the history 
Signed-off-by: chuanyun.lcy <chuanyun.lcy@alibaba-inc.com>
Co-authored-by: chuanyun.lcy <chuanyun.lcy@alibaba-inc.com>
  • Loading branch information
@shaloulcy
shaloulcy and chuanyun.lcy committed Feb 28, 2024
1 parent 5d5e409 commit fcadf29
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 27 deletions.
11 changes: 11 additions & 0 deletions apis/extension/elastic_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request"
AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys"
AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource"
AnnotationUnschedulableResource = QuotaKoordinatorPrefix + "/unschedulable-resource"
AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed"
AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated"
Expand Down Expand Up @@ -195,3 +196,13 @@ func GetChildRequest(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error)
}
return request, nil
}

func GetUnschedulableResource(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
unschedulable := corev1.ResourceList{}
if quota.Annotations[AnnotationUnschedulableResource] != "" {
if err := json.Unmarshal([]byte(quota.Annotations[AnnotationUnschedulableResource]), &unschedulable); err != nil {
return unschedulable, err
}
}
return unschedulable, nil
}
14 changes: 14 additions & 0 deletions pkg/quota-controller/profile/profile_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
quotav1 "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
nodeutil "k8s.io/kubernetes/pkg/util/node"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -138,11 +139,16 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request

// TODO: consider node status.
totalResource := corev1.ResourceList{}
unschedulableResource := corev1.ResourceList{}
for _, node := range nodeList.Items {
totalResource = quotav1.Add(totalResource, GetNodeAllocatable(node))
if node.Spec.Unschedulable || !nodeutil.IsNodeReady(&node) {
unschedulableResource = quotav1.Add(unschedulableResource, GetNodeAllocatable(node))
}
}

decorateTotalResource(profile, totalResource)
decorateTotalResource(profile, unschedulableResource)

resourceKeys := []string{"cpu", "memory"}
raw, ok := profile.Annotations[extension.AnnotationResourceKeys]
Expand Down Expand Up @@ -197,6 +203,14 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
}
quota.Annotations[extension.AnnotationTotalResource] = string(data)

// update unschedulable resource
data, err = json.Marshal(unschedulableResource)
if err != nil {
klog.Errorf("failed marshal unschedulable resources, err: %v", err)
return ctrl.Result{Requeue: true}, err
}
quota.Annotations[extension.AnnotationUnschedulableResource] = string(data)

if !quotaExist {
err = r.Client.Create(context.TODO(), quota)
if err != nil {
Expand Down
90 changes: 63 additions & 27 deletions pkg/quota-controller/profile/profile_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,30 @@ func defaultCreateNode(nodeName string, labels map[string]string, capacity corev
},
Status: corev1.NodeStatus{
Allocatable: capacity,
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
},
}
}

func defaultCreateUnreadyNode(nodeName string, labels map[string]string, capacity corev1.ResourceList) *corev1.Node {
return &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
Labels: labels,
},
Status: corev1.NodeStatus{
Allocatable: capacity,
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionFalse,
},
},
},
}
}
Expand All @@ -77,7 +101,7 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {

nodes := []*corev1.Node{
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateUnreadyNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceListWithStorage(10, 1000, 1000)),
}

Expand All @@ -87,12 +111,13 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
resourceRatio := "0.9"

tests := []struct {
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectQuotaLabels map[string]string
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectUnschedulableResource corev1.ResourceList
expectQuotaLabels map[string]string
}{
{
name: "cn-hangzhou-a profile",
Expand All @@ -107,9 +132,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -129,9 +155,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
expectUnschedulableResource: corev1.ResourceList{},
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile2",
extension.LabelQuotaTreeID: treeID2,
Expand All @@ -154,9 +181,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand Down Expand Up @@ -189,8 +217,9 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
Min: createResourceList(5, 50),
},
},
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -213,9 +242,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -239,9 +269,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
Expand All @@ -268,9 +299,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
Expand Down Expand Up @@ -310,8 +342,12 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
err = json.Unmarshal([]byte(quota.Annotations[extension.AnnotationTotalResource]), &total)
assert.NoError(t, err)

unschedulable, err := extension.GetUnschedulableResource(quota)
assert.NoError(t, err)

assert.True(t, quotav1.Equals(tc.expectQuotaMin, quota.Spec.Min))
assert.True(t, quotav1.Equals(tc.expectTotalResource, total))
assert.True(t, quotav1.Equals(tc.expectUnschedulableResource, unschedulable))
assert.Equal(t, tc.expectQuotaLabels, quota.Labels)
})
}
Expand Down
9 changes: 9 additions & 0 deletions pkg/scheduler/plugins/elasticquota/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,15 @@ func syncElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoS
extension.AnnotationNonPreemptibleRequest: summary.NonPreemptibleRequest,
extension.AnnotationNonPreemptibleUsed: summary.NonPreemptibleUsed,
}

// record the unschedulable resource
if extension.IsTreeRootQuota(eq) {
unschedulable, err := extension.GetUnschedulableResource(eq)
if err == nil {
m[extension.AnnotationUnschedulableResource] = unschedulable
}
}

for k, v := range m {
decorateResource(eq, v)

Expand Down
10 changes: 10 additions & 0 deletions pkg/scheduler/plugins/elasticquota/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,14 @@ func Test_updateElasticQuotaStatusIfChanged(t *testing.T) {

func Test_syncElasticQuotaMetrics(t *testing.T) {
eq := &v1alpha1.ElasticQuota{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"quota.scheduling.koordinator.sh/is-root": "true",
},
Annotations: map[string]string{
"quota.scheduling.koordinator.sh/unschedulable-resource": `{"cpu":"4","memory":"8"}`,
},
},
Spec: v1alpha1.ElasticQuotaSpec{
Max: MakeResourceList().CPU(4).Mem(200).Obj(),
Min: MakeResourceList().CPU(2).Mem(100).Obj(),
Expand Down Expand Up @@ -435,6 +443,8 @@ loopChan:
`label:<name:"field" value:"request" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
`label:<name:"field" value:"runtime" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:5000 > `,
`label:<name:"field" value:"runtime" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
`label:<name:"field" value:"unschedulable-resource" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:4000 > `,
`label:<name:"field" value:"unschedulable-resource" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:8 > `,
`label:<name:"field" value:"used" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:1000 > `,
`label:<name:"field" value:"used" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
}
Expand Down

0 comments on commit fcadf29

Please sign in to comment.