From 36519412f3c797d0fed913d82eae09a6ce863ca8 Mon Sep 17 00:00:00 2001 From: j4ckstraw Date: Mon, 27 Nov 2023 10:53:10 +0800 Subject: [PATCH] fix: kill container after calling eviction api success Signed-off-by: j4ckstraw --- pkg/koordlet/qosmanager/framework/context.go | 8 +++++--- .../qosmanager/plugins/cpuevict/cpu_evict.go | 20 +++++++++---------- .../plugins/memoryevict/memory_evict.go | 15 +++++++------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pkg/koordlet/qosmanager/framework/context.go b/pkg/koordlet/qosmanager/framework/context.go index f73c49c981..794e572d26 100644 --- a/pkg/koordlet/qosmanager/framework/context.go +++ b/pkg/koordlet/qosmanager/framework/context.go @@ -62,20 +62,22 @@ func (r *Evictor) Start(stopCh <-chan struct{}) error { func (r *Evictor) EvictPodsIfNotEvicted(evictPods []*corev1.Pod, node *corev1.Node, reason string, message string) { for _, evictPod := range evictPods { - r.evictPodIfNotEvicted(evictPod, node, reason, message) + r.EvictPodIfNotEvicted(evictPod, node, reason, message) } } -func (r *Evictor) evictPodIfNotEvicted(evictPod *corev1.Pod, node *corev1.Node, reason string, message string) { +func (r *Evictor) EvictPodIfNotEvicted(evictPod *corev1.Pod, node *corev1.Node, reason string, message string) bool { _, evicted := r.podsEvicted.Get(string(evictPod.UID)) if evicted { klog.V(5).Infof("Pod has been evicted! podID: %v, evict reason: %s", evictPod.UID, reason) - return + return true } success := r.evictPod(evictPod, reason, message) if success { _ = r.podsEvicted.SetDefault(string(evictPod.UID), evictPod.UID) } + + return success } func (r *Evictor) evictPod(evictPod *corev1.Pod, reason string, message string) bool { diff --git a/pkg/koordlet/qosmanager/plugins/cpuevict/cpu_evict.go b/pkg/koordlet/qosmanager/plugins/cpuevict/cpu_evict.go index 996f8a0f40..48277cfd94 100644 --- a/pkg/koordlet/qosmanager/plugins/cpuevict/cpu_evict.go +++ b/pkg/koordlet/qosmanager/plugins/cpuevict/cpu_evict.go @@ -292,24 +292,24 @@ func (c *cpuEvictor) killAndEvictBEPodsRelease(node *corev1.Node, bePodInfos []* node.Name, cpuNeedMilliRelease) cpuMilliReleased := int64(0) - var killedPods []*corev1.Pod + hasKillPods := false for _, bePod := range bePodInfos { if cpuMilliReleased >= cpuNeedMilliRelease { break } - podKillMsg := fmt.Sprintf("%s, kill pod: %s", message, util.GetPodKey(bePod.pod)) - helpers.KillContainers(bePod.pod, podKillMsg) + ok := c.evictor.EvictPodIfNotEvicted(bePod.pod, node, resourceexecutor.EvictPodByBECPUSatisfaction, message) + if ok { + podKillMsg := fmt.Sprintf("%s, kill pod: %s", message, util.GetPodKey(bePod.pod)) + helpers.KillContainers(bePod.pod, podKillMsg) - killedPods = append(killedPods, bePod.pod) - cpuMilliReleased = cpuMilliReleased + bePod.milliRequest - - klog.V(5).Infof("cpuEvict pick pod %s/%s to evict", util.GetPodKey(bePod.pod)) + cpuMilliReleased = cpuMilliReleased + bePod.milliRequest + klog.V(5).Infof("cpuEvict pick pod %s/%s to evict", util.GetPodKey(bePod.pod)) + hasKillPods = true + } } - c.evictor.EvictPodsIfNotEvicted(killedPods, node, resourceexecutor.EvictPodByBECPUSatisfaction, message) - - if len(killedPods) > 0 { + if hasKillPods { c.lastEvictTime = time.Now() } klog.V(5).Infof("killAndEvictBEPodsRelease finished! cpuNeedMilliRelease(%d) cpuMilliReleased(%d)", diff --git a/pkg/koordlet/qosmanager/plugins/memoryevict/memory_evict.go b/pkg/koordlet/qosmanager/plugins/memoryevict/memory_evict.go index 72616337c6..89bb350a08 100644 --- a/pkg/koordlet/qosmanager/plugins/memoryevict/memory_evict.go +++ b/pkg/koordlet/qosmanager/plugins/memoryevict/memory_evict.go @@ -165,22 +165,21 @@ func (m *memoryEvictor) killAndEvictBEPods(node *corev1.Node, podMetrics map[str message := fmt.Sprintf("killAndEvictBEPods for node, need to release memory: %v", memoryNeedRelease) memoryReleased := int64(0) - var killedPods []*corev1.Pod for _, bePod := range bePodInfos { if memoryReleased >= memoryNeedRelease { break } - killMsg := fmt.Sprintf("%v, kill pod: %v", message, bePod.pod.Name) - helpers.KillContainers(bePod.pod, killMsg) - killedPods = append(killedPods, bePod.pod) - if bePod.memUsed != 0 { - memoryReleased += int64(bePod.memUsed) + ok := m.evictor.EvictPodIfNotEvicted(bePod.pod, node, resourceexecutor.EvictPodByNodeMemoryUsage, message) + if ok { + killMsg := fmt.Sprintf("%v, kill pod: %v", message, bePod.pod.Name) + helpers.KillContainers(bePod.pod, killMsg) + if bePod.memUsed != 0 { + memoryReleased += int64(bePod.memUsed) + } } } - m.evictor.EvictPodsIfNotEvicted(killedPods, node, resourceexecutor.EvictPodByNodeMemoryUsage, message) - m.lastEvictTime = time.Now() klog.Infof("killAndEvictBEPods completed, memoryNeedRelease(%v) memoryReleased(%v)", memoryNeedRelease, memoryReleased) }