Skip to content

Commit

Permalink
convert miks oversale annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
@bogo-y
bogo-y authored and 王利杰 committed Mar 26, 2024
1 parent 60e7b46 commit 6c3bde1
Show file tree
Hide file tree
Showing 4 changed files with 229 additions and 39 deletions.
2 changes: 2 additions & 0 deletions pkg/descheduler/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (

podutil "github.com/koordinator-sh/koordinator/pkg/descheduler/pod"
"github.com/koordinator-sh/koordinator/pkg/descheduler/utils"
"github.com/koordinator-sh/koordinator/pkg/util"
)

// ReadyNodes returns ready nodes irrespective of whether they are
Expand Down Expand Up @@ -71,6 +72,7 @@ func ReadyNodes(ctx context.Context, client clientset.Interface, nodeInformer co
for _, node := range nodes {
if IsReady(node) {
readyNodes = append(readyNodes, node)
util.RewriteAllocatable(node)
}
}
return readyNodes, nil
Expand Down
45 changes: 6 additions & 39 deletions pkg/slo-controller/noderesource/plugins/batchresource/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package batchresource

import (
"fmt"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -267,51 +266,19 @@ func getNodeAllocatable(node *corev1.Node) corev1.ResourceList {
return getResourceListForCPUAndMemory(node.Status.Allocatable)
}

const oversaleKey = "xiaomi.oversale/physical-resource"

// for xiaomi oversale only
// if oversale component installed, annotation will be added.
// xiaomi.oversale/physical-resource: cpu=32000m,memory=269906472960
func getRealCPUAndMemoryFromAnnotation(node *corev1.Node) corev1.ResourceList {
parseOversaleAnnotation := func(v string) corev1.ResourceList {
result := corev1.ResourceList{}
kvs := strings.Split(v, ",")
for _, kv := range kvs {
kv = strings.TrimSpace(kv)
s := strings.SplitN(kv, "=", 2)
k, v := s[0], s[1]
if k == "cpu" {
cpu, err := resource.ParseQuantity(v)
if err != nil {
klog.V(2).Infof("parse oversale cpu error: %v, value: %v", err, v)
}
result[corev1.ResourceCPU] = cpu
} else if k == "memory" {
memory, err := resource.ParseQuantity(v)
if err != nil {
klog.V(2).Infof("parse oversale memory error: %v, value: %v", err, v)
}
result[corev1.ResourceMemory] = memory
} else {
klog.V(4).Infof("not support key: %s, value: %s", k, v)
}
}
return result
}
result, err := util.GetRealCPUAndMemoryFromOversaleAnnotation(node)
if err != nil {
klog.Warningf("get real CPU and Memory error: %v", err)

if value, exist := node.Annotations[oversaleKey]; exist {
result := parseOversaleAnnotation(value)
cpu, memory := result[corev1.ResourceCPU], result[corev1.ResourceMemory]
if cpu.IsZero() || memory.IsZero() {
klog.Warning("read from annotation `xiaomi.oversale/physical-resource` successful, but it's zero, fallback to allocatable")
return getResourceListForCPUAndMemory(node.Status.Allocatable)
}

return result
// backoff allocatable
return getResourceListForCPUAndMemory(node.Status.Allocatable)
}

// backoff allocatable
return getResourceListForCPUAndMemory(node.Status.Allocatable)
return result
}

// getNodeReservation gets node-level safe-guarding reservation with the node's allocatable
Expand Down
67 changes: 67 additions & 0 deletions pkg/util/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"strconv"
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -136,3 +137,69 @@ func GetNodeAnnoReservedJson(reserved apiext.NodeReservation) string {

return result
}

func GetResourceListForCPUAndMemory(rl corev1.ResourceList) corev1.ResourceList {
return quotav1.Mask(rl, []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory})
}

// for xiaomi oversale only
// if oversale component installed, annotation will be added.
// xiaomi.oversale/physical-resource: cpu=32000m,memory=269906472960
func parseOversaleAnnotation(v string) corev1.ResourceList {
result := corev1.ResourceList{}
kvs := strings.Split(v, ",")
for _, kv := range kvs {
kv = strings.TrimSpace(kv)
s := strings.Split(kv, "=")
if len(s) != 2 {
continue
}
k, v := s[0], s[1]
if k == "cpu" {
cpu, err := resource.ParseQuantity(v)
if err != nil {
klog.V(2).Infof("parse oversale cpu error: %v, value: %v", err, v)
}
result[corev1.ResourceCPU] = cpu
} else if k == "memory" {
memory, err := resource.ParseQuantity(v)
if err != nil {
klog.V(2).Infof("parse oversale memory error: %v, value: %v", err, v)
}
result[corev1.ResourceMemory] = memory
} else {
klog.V(4).Infof("not support key: %s, value: %s", k, v)
}
}
return result
}

const oversaleKey = "xiaomi.oversale/physical-resource"

func GetRealCPUAndMemoryFromOversaleAnnotation(node *corev1.Node) (corev1.ResourceList, error) {
if value, exist := node.Annotations[oversaleKey]; exist {
result := parseOversaleAnnotation(value)
cpu, memory := result[corev1.ResourceCPU], result[corev1.ResourceMemory]
if cpu.IsZero() || memory.IsZero() {
klog.V(6).Infof("read from annotation `xiaomi.oversale/physical-resource` successful, but it's zero, fallback to allocatable")
return nil, fmt.Errorf("zero value found from oversale annotation")
}

return result, nil
}

// return nil if not found annotation
return nil, fmt.Errorf("annotation: %v not found", oversaleKey)
}
func RewriteAllocatable(node *corev1.Node) {
result, err := GetRealCPUAndMemoryFromOversaleAnnotation(node)
if err != nil {
klog.Warning("get real CPU and Memory error: %v, skip rewrite node %v allocatable", err.Error(), node.Name)
return
}

for _, resourceName := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} {
klog.V(6).Infof("rewrite node %v resource name %v to %v", node.Name, resourceName, result[resourceName])
node.Status.Allocatable[resourceName] = result[resourceName]
}
}
154 changes: 154 additions & 0 deletions pkg/util/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package util

import (
"encoding/json"
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"reflect"
"testing"

Expand Down Expand Up @@ -309,3 +311,155 @@ func TestTrimNodeAllocatableByNodeReservation(t *testing.T) {
})
}
}

func Test_rewriteAllocatable(t *testing.T) {
testCases := []struct {
name string
node *corev1.Node
expectAllocatable corev1.ResourceList
}{
{
name: "oversale-annotation-not-exist",
node: &corev1.Node{
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
{
name: "oversale-annotation-format-error",
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
oversaleKey: "cpu==1,memory=1Gi",
},
},
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
{
name: "oversale-annotation-format-error",
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
oversaleKey: "mem=1Gi,cpu=1",
},
},
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
{
name: "oversale-annotation-format-error",
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
oversaleKey: ",memory=1GI,cpu=1",
},
},
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
corev1.ResourceMemory: resource.MustParse("2Gi"),
},
},
{
name: "oversale-cpu-not-exist",
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
oversaleKey: "cpu=1, memory=1081007022080",
},
},
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("233Gi"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1081007022080"),
},
},
{
name: "oversale-memory-not-exist",
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
oversaleKey: "memory=1Gi,,,cpu=128000m,,,",
},
},
Status: corev1.NodeStatus{
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("256000m"),
},
},
},
expectAllocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("128000m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
RewriteAllocatable(tc.node)
errs := validateNode(tc.node, tc.expectAllocatable)
assert.Nil(t, errs)
t.Log(errs)
t.Logf("node: %v", tc.node)
})
}
}

func validateNode(node *corev1.Node, expect corev1.ResourceList) []error {
errs := make([]error, 0)

if len(node.Status.Allocatable) == 0 {
errs = append(errs, fmt.Errorf("node allocatable resources are empty"))
}
if len(node.Status.Allocatable) != len(expect) {
errs = append(errs, fmt.Errorf("unexpected length"))
return errs
}
for k := range expect {
if node.Status.Allocatable[k] != expect[k] {
errs = append(errs, fmt.Errorf("unexpected value"))
}
}
if len(errs) == 0 {
return nil
}
return errs
}

0 comments on commit 6c3bde1

Please sign in to comment.