Skip to content

Commit

Permalink
fix: kill container after calling eviction api success
Browse files Browse the repository at this point in the history
Signed-off-by: j4ckstraw <j4ckstraw@foxmail.com>
  • Loading branch information
j4ckstraw committed Nov 27, 2023
1 parent 3784df1 commit 3651941
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 21 deletions.
8 changes: 5 additions & 3 deletions pkg/koordlet/qosmanager/framework/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,20 +62,22 @@ func (r *Evictor) Start(stopCh <-chan struct{}) error {

func (r *Evictor) EvictPodsIfNotEvicted(evictPods []*corev1.Pod, node *corev1.Node, reason string, message string) {
for _, evictPod := range evictPods {
r.evictPodIfNotEvicted(evictPod, node, reason, message)
r.EvictPodIfNotEvicted(evictPod, node, reason, message)
}
}

func (r *Evictor) evictPodIfNotEvicted(evictPod *corev1.Pod, node *corev1.Node, reason string, message string) {
func (r *Evictor) EvictPodIfNotEvicted(evictPod *corev1.Pod, node *corev1.Node, reason string, message string) bool {
_, evicted := r.podsEvicted.Get(string(evictPod.UID))
if evicted {
klog.V(5).Infof("Pod has been evicted! podID: %v, evict reason: %s", evictPod.UID, reason)
return
return true
}
success := r.evictPod(evictPod, reason, message)
if success {
_ = r.podsEvicted.SetDefault(string(evictPod.UID), evictPod.UID)
}

return success
}

func (r *Evictor) evictPod(evictPod *corev1.Pod, reason string, message string) bool {
Expand Down
20 changes: 10 additions & 10 deletions pkg/koordlet/qosmanager/plugins/cpuevict/cpu_evict.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,24 +292,24 @@ func (c *cpuEvictor) killAndEvictBEPodsRelease(node *corev1.Node, bePodInfos []*
node.Name, cpuNeedMilliRelease)

cpuMilliReleased := int64(0)
var killedPods []*corev1.Pod
hasKillPods := false
for _, bePod := range bePodInfos {
if cpuMilliReleased >= cpuNeedMilliRelease {
break
}

podKillMsg := fmt.Sprintf("%s, kill pod: %s", message, util.GetPodKey(bePod.pod))
helpers.KillContainers(bePod.pod, podKillMsg)
ok := c.evictor.EvictPodIfNotEvicted(bePod.pod, node, resourceexecutor.EvictPodByBECPUSatisfaction, message)
if ok {
podKillMsg := fmt.Sprintf("%s, kill pod: %s", message, util.GetPodKey(bePod.pod))
helpers.KillContainers(bePod.pod, podKillMsg)

killedPods = append(killedPods, bePod.pod)
cpuMilliReleased = cpuMilliReleased + bePod.milliRequest

klog.V(5).Infof("cpuEvict pick pod %s/%s to evict", util.GetPodKey(bePod.pod))
cpuMilliReleased = cpuMilliReleased + bePod.milliRequest
klog.V(5).Infof("cpuEvict pick pod %s/%s to evict", util.GetPodKey(bePod.pod))
hasKillPods = true
}
}

c.evictor.EvictPodsIfNotEvicted(killedPods, node, resourceexecutor.EvictPodByBECPUSatisfaction, message)

if len(killedPods) > 0 {
if hasKillPods {
c.lastEvictTime = time.Now()
}
klog.V(5).Infof("killAndEvictBEPodsRelease finished! cpuNeedMilliRelease(%d) cpuMilliReleased(%d)",
Expand Down
15 changes: 7 additions & 8 deletions pkg/koordlet/qosmanager/plugins/memoryevict/memory_evict.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,22 +165,21 @@ func (m *memoryEvictor) killAndEvictBEPods(node *corev1.Node, podMetrics map[str
message := fmt.Sprintf("killAndEvictBEPods for node, need to release memory: %v", memoryNeedRelease)
memoryReleased := int64(0)

var killedPods []*corev1.Pod
for _, bePod := range bePodInfos {
if memoryReleased >= memoryNeedRelease {
break
}

killMsg := fmt.Sprintf("%v, kill pod: %v", message, bePod.pod.Name)
helpers.KillContainers(bePod.pod, killMsg)
killedPods = append(killedPods, bePod.pod)
if bePod.memUsed != 0 {
memoryReleased += int64(bePod.memUsed)
ok := m.evictor.EvictPodIfNotEvicted(bePod.pod, node, resourceexecutor.EvictPodByNodeMemoryUsage, message)
if ok {
killMsg := fmt.Sprintf("%v, kill pod: %v", message, bePod.pod.Name)
helpers.KillContainers(bePod.pod, killMsg)
if bePod.memUsed != 0 {
memoryReleased += int64(bePod.memUsed)
}
}
}

m.evictor.EvictPodsIfNotEvicted(killedPods, node, resourceexecutor.EvictPodByNodeMemoryUsage, message)

m.lastEvictTime = time.Now()
klog.Infof("killAndEvictBEPods completed, memoryNeedRelease(%v) memoryReleased(%v)", memoryNeedRelease, memoryReleased)
}
Expand Down

0 comments on commit 3651941

Please sign in to comment.