diff --git a/pkg/descheduler/framework/plugins/loadaware/utilization_util.go b/pkg/descheduler/framework/plugins/loadaware/utilization_util.go index c844e3f141..69112bddc8 100644 --- a/pkg/descheduler/framework/plugins/loadaware/utilization_util.go +++ b/pkg/descheduler/framework/plugins/loadaware/utilization_util.go @@ -435,7 +435,8 @@ func balancePods(ctx context.Context, klog.V(4).InfoS("No removable pods on node, try next node", "node", klog.KObj(srcNode.node), "nodePool", nodePoolName) continue } - sortPodsOnOneOverloadedNode(srcNode, removablePods, resourceWeights) + sortPodsOnOneOverloadedNode(srcNode, removablePods, resourceWeights, prod) + evictPods(ctx, nodePoolName, dryRun, prod, removablePods, srcNode, totalAvailableUsages, podEvictor, podFilter, continueEviction, evictionReasonGenerator) } } @@ -665,14 +666,28 @@ func calcAverageResourceUsagePercent(nodeUsages map[string]*NodeUsage) (Resource } return average, prodAverage } -func sortPodsOnOneOverloadedNode(srcNode NodeInfo, removablePods []*corev1.Pod, resourceWeights map[corev1.ResourceName]int64) { +func sortPodsOnOneOverloadedNode(srcNode NodeInfo, removablePods []*corev1.Pod, resourceWeights map[corev1.ResourceName]int64, prod bool) { weights := make(map[corev1.ResourceName]int64) // get the overused resource of this node, and the weights of appropriately using resources will be zero. - overusedResources, _ := isNodeOverutilized(srcNode.usage, srcNode.thresholds.highResourceThreshold) - for or := range overusedResources { + var overusedResources corev1.ResourceList + if prod { + overusedResources, _ = isNodeOverutilized(srcNode.prodUsage, srcNode.thresholds.prodHighResourceThreshold) + } else { + overusedResources, _ = isNodeOverutilized(srcNode.usage, srcNode.thresholds.highResourceThreshold) + } + resourcesThatExceedThresholds := map[corev1.ResourceName]resource.Quantity{} + for or, used := range overusedResources { + usedCopy := used.DeepCopy() weights[or] = resourceWeights[or] + if prod { + usedCopy.Sub(*srcNode.thresholds.prodHighResourceThreshold[or]) + } else { + usedCopy.Sub(*srcNode.thresholds.highResourceThreshold[or]) + } + resourcesThatExceedThresholds[or] = usedCopy } sorter.SortPodsByUsage( + resourcesThatExceedThresholds, removablePods, srcNode.podMetrics, map[string]corev1.ResourceList{srcNode.node.Name: srcNode.node.Status.Allocatable}, diff --git a/pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go b/pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go index 2f6ebf2254..990c956e88 100644 --- a/pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go +++ b/pkg/descheduler/framework/plugins/loadaware/utilization_util_test.go @@ -226,6 +226,6 @@ func TestSortPodsOnOneOverloadedNode(t *testing.T) { corev1.ResourceCPU: int64(1), corev1.ResourceMemory: int64(1), } - sortPodsOnOneOverloadedNode(nodeInfo, removablePods, resourceWeights) + sortPodsOnOneOverloadedNode(nodeInfo, removablePods, resourceWeights, false) assert.Equal(t, expectedResult, removablePods) } diff --git a/pkg/descheduler/utils/sorter/pod.go b/pkg/descheduler/utils/sorter/pod.go index 013c121414..e1f37f7e34 100644 --- a/pkg/descheduler/utils/sorter/pod.go +++ b/pkg/descheduler/utils/sorter/pod.go @@ -18,6 +18,7 @@ package sorter import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" schedulingcorev1helper "k8s.io/component-helpers/scheduling/corev1" apiscorehelper "k8s.io/kubernetes/pkg/apis/core/helper" @@ -103,16 +104,17 @@ func KoordinatorPriorityClass(p1, p2 *corev1.Pod) int { } // PodUsage compares pods by the actual usage -func PodUsage(podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) CompareFn { - scorer := ResourceUsageScorer(resourceToWeightMap) +func PodUsage(resourcesThatExceedThresholds map[corev1.ResourceName]resource.Quantity, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, resourceToWeightMap map[corev1.ResourceName]int64) CompareFn { + scorer := ResourceUsageScorerPod(resourceToWeightMap) return func(p1, p2 *corev1.Pod) int { p1Metric, p1Found := podMetrics[types.NamespacedName{Namespace: p1.Namespace, Name: p1.Name}] p2Metric, p2Found := podMetrics[types.NamespacedName{Namespace: p2.Namespace, Name: p2.Name}] if !p1Found || !p2Found { return cmpBool(!p1Found, !p2Found) } - p1Score := scorer(p1Metric.ResourceList, nodeAllocatableMap[p1.Spec.NodeName]) - p2Score := scorer(p2Metric.ResourceList, nodeAllocatableMap[p2.Spec.NodeName]) + p1Score := scorer(p1Metric.ResourceList, resourcesThatExceedThresholds) + p2Score := scorer(p2Metric.ResourceList, resourcesThatExceedThresholds) + if p1Score == p2Score { return 0 } @@ -172,6 +174,6 @@ func PodSorter(cmp ...CompareFn) *MultiSorter { return OrderedBy(comparators...) } -func SortPodsByUsage(pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) { - PodSorter(Reverse(PodUsage(podMetrics, nodeAllocatableMap, resourceToWeightMap))).Sort(pods) +func SortPodsByUsage(resourcesThatExceedThresholds map[corev1.ResourceName]resource.Quantity, pods []*corev1.Pod, podMetrics map[types.NamespacedName]*slov1alpha1.ResourceMap, nodeAllocatableMap map[string]corev1.ResourceList, resourceToWeightMap map[corev1.ResourceName]int64) { + PodSorter(Reverse(PodUsage(resourcesThatExceedThresholds, podMetrics, resourceToWeightMap))).Sort(pods) } diff --git a/pkg/descheduler/utils/sorter/pod_test.go b/pkg/descheduler/utils/sorter/pod_test.go index 4e3c1f4bc7..455278c996 100644 --- a/pkg/descheduler/utils/sorter/pod_test.go +++ b/pkg/descheduler/utils/sorter/pod_test.go @@ -129,8 +129,69 @@ func TestSortPods(t *testing.T) { corev1.ResourceMemory: 1, corev1.ResourcePods: 1, } - SortPodsByUsage(pods, podMetrics, nodeAllocatableMap, resourceToWeightMap) - expectedPodsOrder := []string{"test-1", "test-12", "test-18", "test-19", "test-17", "test-16", "test-15", "test-21", "test-20", "test-23", "test-22", "test-9", "test-8", "test-11", "test-10", "test-13", "test-14", "test-2", "test-3", "test-7", "test-4", "test-6", "test-5"} + SortPodsByUsage(nil, pods, podMetrics, nodeAllocatableMap, resourceToWeightMap) + expectedPodsOrder := []string{"test-1", "test-12", "test-18", "test-16", "test-19", "test-17", "test-15", "test-21", "test-20", "test-23", "test-22", "test-9", "test-8", "test-11", "test-10", "test-13", "test-14", "test-2", "test-3", "test-7", "test-4", "test-6", "test-5"} + var podsOrder []string + for _, v := range pods { + podsOrder = append(podsOrder, v.Name) + } + assert.Equal(t, expectedPodsOrder, podsOrder) +} + +func TestSortPods2(t *testing.T) { + creationTime := time.Now() + pods := []*corev1.Pod{ + makePod("test-1", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime), + makePod("test-2", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime), + makePod("test-3", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime), + makePod("test-4", extension.PriorityBatchValueMin, extension.QoSBE, corev1.PodQOSBestEffort, creationTime), + } + podMetrics := map[types.NamespacedName]*slov1alpha1.ResourceMap{ + {Namespace: "default", Name: "test-1"}: { + ResourceList: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("5Gi"), + }, + }, + {Namespace: "default", Name: "test-2"}: { + ResourceList: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + {Namespace: "default", Name: "test-3"}: { + ResourceList: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("3Gi"), + }, + }, + {Namespace: "default", Name: "test-4"}: { + ResourceList: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + } + + nodeAllocatableMap := map[string]corev1.ResourceList{ + "test-node": { + corev1.ResourceCPU: resource.MustParse("96"), + corev1.ResourceMemory: resource.MustParse("512Gi"), + }, + } + + resourcesThatExceedThresholds := map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + } + + resourceToWeightMap := map[corev1.ResourceName]int64{ + corev1.ResourceCPU: 1, + corev1.ResourceMemory: 2, + corev1.ResourcePods: 1, + } + SortPodsByUsage(resourcesThatExceedThresholds, pods, podMetrics, nodeAllocatableMap, resourceToWeightMap) + expectedPodsOrder := []string{"test-2", "test-1", "test-3", "test-4"} var podsOrder []string for _, v := range pods { podsOrder = append(podsOrder, v.Name) diff --git a/pkg/descheduler/utils/sorter/scorer.go b/pkg/descheduler/utils/sorter/scorer.go index 048e0f350e..9f10ebb020 100644 --- a/pkg/descheduler/utils/sorter/scorer.go +++ b/pkg/descheduler/utils/sorter/scorer.go @@ -37,6 +37,23 @@ func ResourceUsageScorer(resToWeightMap map[corev1.ResourceName]int64) func(requ } } +func ResourceUsageScorerPod(resToWeightMap map[corev1.ResourceName]int64) func(requested corev1.ResourceList, allocatable corev1.ResourceList) float64 { + return func(requested, allocatable corev1.ResourceList) float64 { + var nodeScore float64 + var weightSum int64 + for resourceName, quantity := range requested { + weight := resToWeightMap[resourceName] + resourceScore := mostRequestedScorePod(getResourceValue(resourceName, quantity), getResourceValue(resourceName, allocatable[resourceName])) + nodeScore += resourceScore * float64(weight) + weightSum += weight + } + if weightSum == 0 { + return 0 + } + return nodeScore / float64(weightSum) + } +} + func mostRequestedScore(requested, capacity int64) int64 { if capacity == 0 { return 0 @@ -50,6 +67,19 @@ func mostRequestedScore(requested, capacity int64) int64 { return (requested * 1000) / capacity } +// mostRequestedScorePod The closer the request is to the capacity, the higher the score.The score will be higher when requested >= ratio +func mostRequestedScorePod(requested, capacity int64) float64 { + if capacity == 0 { + return 0 + } + ratio := float64(requested) / float64(capacity) + if ratio >= 1 { + return 1/ratio + 1 + } else { + return ratio + } +} + func getResourceValue(resourceName corev1.ResourceName, quantity resource.Quantity) int64 { if resourceName == corev1.ResourceCPU { return quantity.MilliValue() diff --git a/pkg/descheduler/utils/sorter/scorer_test.go b/pkg/descheduler/utils/sorter/scorer_test.go new file mode 100644 index 0000000000..16b72d6852 --- /dev/null +++ b/pkg/descheduler/utils/sorter/scorer_test.go @@ -0,0 +1,94 @@ +package sorter + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestFunctions(t *testing.T) { + resToWeightMap := map[corev1.ResourceName]int64{ + corev1.ResourceCPU: 2, + corev1.ResourceMemory: 3, + } + + requested1 := corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(1024, resource.BinarySI), + } + allocatable1 := corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(2048, resource.BinarySI), + } + + scorer1 := ResourceUsageScorer(resToWeightMap) + score1 := scorer1(requested1, allocatable1) + if score1 != 500 { + t.Fatal("ResourceUsageScorer score1") + } + + score2 := mostRequestedScore(800, 1000) + if score2 != 800 { + t.Fatal("ResourceUsageScorer score2") + } + score3 := mostRequestedScore(1200, 1000) + if score3 != 1000 { + t.Fatal("ResourceUsageScorer score3") + } + + score4 := mostRequestedScore(1200, 0) + if score4 != 0 { + t.Fatal("ResourceUsageScorer score4") + } + + resToWeightMap = map[corev1.ResourceName]int64{ + corev1.ResourceCPU: 0, + corev1.ResourceMemory: 0, + } + + scorer1 = ResourceUsageScorer(resToWeightMap) + score1 = scorer1(requested1, allocatable1) + if score1 != 0 { + t.Fatal("ResourceUsageScorer score5") + } + + qCPU := resource.NewMilliQuantity(500, resource.DecimalSI) + valueCPU := getResourceValue(corev1.ResourceCPU, *qCPU) + if valueCPU != 500 { + t.Fatal("ResourceUsageScorer score6") + } + + qMem := resource.NewQuantity(1024, resource.BinarySI) + valueMem := getResourceValue(corev1.ResourceMemory, *qMem) + if valueMem != 1024 { + t.Fatal("ResourceUsageScorer score7") + } +} + +func TestResourceUsageScorerPod(t *testing.T) { + resToWeightMap := map[corev1.ResourceName]int64{ + corev1.ResourceCPU: 0, + corev1.ResourceMemory: 0, + } + + scorer1 := ResourceUsageScorerPod(resToWeightMap) + requested1 := corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(1024, resource.BinarySI), + } + allocatable1 := corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewMilliQuantity(1000, resource.DecimalSI), + corev1.ResourceMemory: *resource.NewQuantity(2048, resource.BinarySI), + } + + score1 := scorer1(requested1, allocatable1) + if score1 != 0 { + t.Fatal("TestResourceUsageScorerPod ResourceUsageScorerPod fail") + } + + score := mostRequestedScorePod(100, 0) + if score != 0 { + t.Fatal("TestResourceUsageScorerPod mostRequestedScorePod fail") + } +}