From 02c6cde879fa3ebb575909b5140b7b5debf0a686 Mon Sep 17 00:00:00 2001 From: FillZpp Date: Fri, 15 Jul 2022 11:47:54 +0800 Subject: [PATCH] CloneSet supports to calculate scale number excluding Pods in PreparingDelete Signed-off-by: FillZpp --- apis/apps/v1alpha1/cloneset_types.go | 4 + .../cloneset/sync/cloneset_scale.go | 37 +-- .../cloneset/sync/cloneset_sync_utils.go | 82 ++++-- .../cloneset/sync/cloneset_sync_utils_test.go | 266 ++++++++++++++++-- test/e2e/apps/cloneset.go | 180 +++++++++++- 5 files changed, 494 insertions(+), 75 deletions(-) diff --git a/apis/apps/v1alpha1/cloneset_types.go b/apis/apps/v1alpha1/cloneset_types.go index c31c8430e5..4d95dd224e 100644 --- a/apis/apps/v1alpha1/cloneset_types.go +++ b/apis/apps/v1alpha1/cloneset_types.go @@ -30,6 +30,10 @@ const ( // DefaultCloneSetMaxUnavailable is the default value of maxUnavailable for CloneSet update strategy. DefaultCloneSetMaxUnavailable = "20%" + + // CloneSetScalingExcludePreparingDeleteKey is the label key that enables scalingExcludePreparingDelete + // only for this CloneSet, which means it will calculate scale number excluding Pods in PreparingDelete state. + CloneSetScalingExcludePreparingDeleteKey = "apps.kruise.io/cloneset-scaling-exclude-preparing-delete" ) // CloneSetSpec defines the desired state of CloneSet diff --git a/pkg/controller/cloneset/sync/cloneset_scale.go b/pkg/controller/cloneset/sync/cloneset_scale.go index b696822d4d..69ca12c685 100644 --- a/pkg/controller/cloneset/sync/cloneset_scale.go +++ b/pkg/controller/cloneset/sync/cloneset_scale.go @@ -70,19 +70,16 @@ func (r *realControl) Scale( diffRes := calculateDiffsWithExpectation(updateCS, pods, currentRevision, updateRevision) updatedPods, notUpdatedPods := clonesetutils.SplitPodsByRevision(pods, updateRevision) - if diffRes.scaleNum > 0 && diffRes.scaleNum > diffRes.scaleUpLimit { + if diffRes.scaleUpNum > diffRes.scaleUpLimit { r.recorder.Event(updateCS, v1.EventTypeWarning, "ScaleUpLimited", fmt.Sprintf("scaleUp is limited because of scaleStrategy.maxUnavailable, limit: %d", diffRes.scaleUpLimit)) } // 3. scale out - if diffRes.scaleNum > 0 && diffRes.scaleUpLimit > 0 { + if diffRes.scaleUpNum > 0 { // total number of this creation expectedCreations := diffRes.scaleUpLimit // lack number of current version - expectedCurrentCreations := 0 - if diffRes.scaleNumOldRevision > 0 { - expectedCurrentCreations = diffRes.scaleNumOldRevision - } + expectedCurrentCreations := diffRes.scaleUpNumOldRevision klog.V(3).Infof("CloneSet %s begin to scale out %d pods including %d (current rev)", controllerKey, expectedCreations, expectedCurrentCreations) @@ -110,44 +107,36 @@ func (r *realControl) Scale( // 5. specified delete if podsToDelete := util.DiffPods(podsSpecifiedToDelete, podsInPreDelete); len(podsToDelete) > 0 { newPodsToDelete, oldPodsToDelete := clonesetutils.SplitPodsByRevision(podsToDelete, updateRevision) - klog.V(3).Infof("CloneSet %s try to delete pods specified. Delete ready limit: %d. Pods: %v, %v.", + klog.V(3).Infof("CloneSet %s try to delete pods specified. Delete ready limit: %d. New Pods: %v, old Pods: %v.", controllerKey, diffRes.deleteReadyLimit, util.GetPodNames(newPodsToDelete).List(), util.GetPodNames(oldPodsToDelete).List()) - podsToDelete = make([]*v1.Pod, 0, len(podsToDelete)) - for _, pod := range newPodsToDelete { - if !isPodReady(coreControl, pod) { - podsToDelete = append(podsToDelete, pod) - } else if diffRes.deleteReadyLimit > 0 { - podsToDelete = append(podsToDelete, pod) - diffRes.deleteReadyLimit-- - } - } - for _, pod := range oldPodsToDelete { + podsCanDelete := make([]*v1.Pod, 0, len(podsToDelete)) + for _, pod := range podsToDelete { if !isPodReady(coreControl, pod) { - podsToDelete = append(podsToDelete, pod) + podsCanDelete = append(podsCanDelete, pod) } else if diffRes.deleteReadyLimit > 0 { - podsToDelete = append(podsToDelete, pod) + podsCanDelete = append(podsCanDelete, pod) diffRes.deleteReadyLimit-- } } - if modified, err := r.deletePods(updateCS, podsToDelete, pvcs); err != nil || modified { + if modified, err := r.deletePods(updateCS, podsCanDelete, pvcs); err != nil || modified { return modified, err } } // 6. scale in - if diffRes.scaleNum < 0 { + if diffRes.scaleDownNum > 0 { if numToDelete > 0 { klog.V(3).Infof("CloneSet %s skip to scale in %d for %d to delete, including %d specified and %d preDelete", - controllerKey, diffRes.scaleNum, numToDelete, len(podsSpecifiedToDelete), len(podsInPreDelete)) + controllerKey, diffRes.scaleDownNum, numToDelete, len(podsSpecifiedToDelete), len(podsInPreDelete)) return false, nil } klog.V(3).Infof("CloneSet %s begin to scale in %d pods including %d (current rev), delete ready limit: %d", - controllerKey, -diffRes.scaleNum, -diffRes.scaleNumOldRevision, diffRes.deleteReadyLimit) + controllerKey, diffRes.scaleDownNum, diffRes.scaleDownNumOldRevision, diffRes.deleteReadyLimit) - podsPreparingToDelete := r.choosePodsToDelete(updateCS, -diffRes.scaleNum, -diffRes.scaleNumOldRevision, notUpdatedPods, updatedPods) + podsPreparingToDelete := r.choosePodsToDelete(updateCS, diffRes.scaleDownNum, diffRes.scaleDownNumOldRevision, notUpdatedPods, updatedPods) podsToDelete := make([]*v1.Pod, 0, len(podsPreparingToDelete)) for _, pod := range podsPreparingToDelete { if !isPodReady(coreControl, pod) { diff --git a/pkg/controller/cloneset/sync/cloneset_sync_utils.go b/pkg/controller/cloneset/sync/cloneset_sync_utils.go index 493228c557..3e7714e419 100644 --- a/pkg/controller/cloneset/sync/cloneset_sync_utils.go +++ b/pkg/controller/cloneset/sync/cloneset_sync_utils.go @@ -17,6 +17,7 @@ limitations under the License. package sync import ( + "flag" "math" "reflect" @@ -35,21 +36,35 @@ import ( "k8s.io/utils/integer" ) +func init() { + flag.BoolVar(&scalingExcludePreparingDelete, "cloneset-scaling-exclude-preparing-delete", false, + "If true, CloneSet Controller will calculate scale number excluding Pods in PreparingDelete state.") +} + +var ( + // scalingExcludePreparingDelete indicates whether the controller should calculate scale number excluding Pods in PreparingDelete state. + scalingExcludePreparingDelete bool +) + type expectationDiffs struct { - // scaleNum is the diff number that should scale - // '0' means no need to scale - // positive number means need to scale out - // negative number means need to scale in - scaleNum int - // scaleNumOldRevision is part of the scaleNum number - // it indicates the scale number of old revision Pods - scaleNumOldRevision int + // scaleUpNum is a non-negative integer, which indicates the number that should scale up. + scaleUpNum int + // scaleNumOldRevision is a non-negative integer, which indicates the number of old revision Pods that should scale up. + // It might be bigger than scaleUpNum, but controller will scale up at most scaleUpNum number of Pods. + scaleUpNumOldRevision int + // scaleDownNum is a non-negative integer, which indicates the number that should scale down. + scaleDownNum int + // scaleDownNumOldRevision is a non-negative integer, which indicates the number of old revision Pods that should scale down. + // It might be bigger than scaleDownNum, but controller will scale down at most scaleDownNum number of Pods. + scaleDownNumOldRevision int + // scaleUpLimit is the limit number of creating Pods when scaling up // it is limited by scaleStrategy.maxUnavailable scaleUpLimit int // deleteReadyLimit is the limit number of ready Pods that can be deleted // it is limited by UpdateStrategy.maxUnavailable deleteReadyLimit int + // useSurge is the number that temporarily expect to be above the desired replicas useSurge int // useSurgeOldRevision is part of the useSurge number @@ -93,7 +108,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu var newRevisionCount, newRevisionActiveCount, oldRevisionCount, oldRevisionActiveCount int var unavailableNewRevisionCount, unavailableOldRevisionCount int - var toDeleteNewRevisionCount, toDeleteOldRevisionCount, preDeletingCount int + var toDeleteNewRevisionCount, toDeleteOldRevisionCount, preDeletingNewRevisionCount, preDeletingOldRevisionCount int defer func() { if res.isEmpty() { return @@ -101,12 +116,12 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu klog.V(1).Infof("Calculate diffs for CloneSet %s/%s, replicas=%d, partition=%d, maxSurge=%d, maxUnavailable=%d,"+ " allPods=%d, newRevisionPods=%d, newRevisionActivePods=%d, oldRevisionPods=%d, oldRevisionActivePods=%d,"+ " unavailableNewRevisionCount=%d, unavailableOldRevisionCount=%d,"+ - " preDeletingCount=%d, toDeleteNewRevisionCount=%d, toDeleteOldRevisionCount=%d."+ + " preDeletingNewRevisionCount=%d, preDeletingOldRevisionCount=%d, toDeleteNewRevisionCount=%d, toDeleteOldRevisionCount=%d."+ " Result: %+v", cs.Namespace, cs.Name, replicas, partition, maxSurge, maxUnavailable, len(pods), newRevisionCount, newRevisionActiveCount, oldRevisionCount, oldRevisionActiveCount, unavailableNewRevisionCount, unavailableOldRevisionCount, - preDeletingCount, toDeleteNewRevisionCount, toDeleteOldRevisionCount, + preDeletingNewRevisionCount, preDeletingOldRevisionCount, toDeleteNewRevisionCount, toDeleteOldRevisionCount, res) }() @@ -116,7 +131,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu switch state := lifecycle.GetPodLifecycleState(p); state { case appspub.LifecycleStatePreparingDelete: - preDeletingCount++ + preDeletingNewRevisionCount++ default: newRevisionActiveCount++ @@ -132,7 +147,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu switch state := lifecycle.GetPodLifecycleState(p); state { case appspub.LifecycleStatePreparingDelete: - preDeletingCount++ + preDeletingOldRevisionCount++ default: oldRevisionActiveCount++ @@ -147,7 +162,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu updateOldDiff := oldRevisionActiveCount - partition updateNewDiff := newRevisionActiveCount - (replicas - partition) - totalUnavailable := preDeletingCount + unavailableNewRevisionCount + unavailableOldRevisionCount + totalUnavailable := preDeletingNewRevisionCount + preDeletingOldRevisionCount + unavailableNewRevisionCount + unavailableOldRevisionCount // If the currentRevision and updateRevision are consistent, Pods can only update to this revision // If the CloneSetPartitionRollback is not enabled, Pods can only update to the new revision if updateRevision == currentRevision || !utilfeature.DefaultFeatureGate.Enabled(features.CloneSetPartitionRollback) { @@ -161,7 +176,7 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu // Use surge for maxUnavailable not satisfied before scaling var scaleSurge, scaleOldRevisionSurge int if toDeleteCount := toDeleteNewRevisionCount + toDeleteOldRevisionCount; toDeleteCount > 0 { - scaleSurge = integer.IntMin(integer.IntMax((unavailableNewRevisionCount+unavailableOldRevisionCount+toDeleteCount+preDeletingCount)-maxUnavailable, 0), toDeleteCount) + scaleSurge = integer.IntMin(integer.IntMax((totalUnavailable+toDeleteCount)-maxUnavailable, 0), toDeleteCount) if scaleSurge > toDeleteNewRevisionCount { scaleOldRevisionSurge = scaleSurge - toDeleteNewRevisionCount } @@ -193,19 +208,34 @@ func calculateDiffsWithExpectation(cs *appsv1alpha1.CloneSet, pods []*v1.Pod, cu } } - res.scaleNum = replicas + res.useSurge - len(pods) - if res.scaleNum > 0 { - res.scaleNumOldRevision = integer.IntMax(partition+res.useSurgeOldRevision-oldRevisionCount, 0) - } else if res.scaleNum < 0 { - res.scaleNumOldRevision = integer.IntMin(partition+res.useSurgeOldRevision-oldRevisionCount, 0) + // prepare for scale calculation + scaleUpTotalCount := len(pods) + scaleDownTotalCount := len(pods) - toDeleteOldRevisionCount - toDeleteNewRevisionCount + scaleUpTotalOldCount := oldRevisionCount + scaleDownTotalOldCount := oldRevisionCount - toDeleteOldRevisionCount + if shouldScalingExcludePreparingDelete(cs) { + scaleUpTotalCount = scaleUpTotalCount - preDeletingOldRevisionCount - preDeletingNewRevisionCount + scaleDownTotalCount = scaleDownTotalCount - preDeletingOldRevisionCount - preDeletingNewRevisionCount + scaleUpTotalOldCount = scaleUpTotalOldCount - preDeletingOldRevisionCount + scaleDownTotalOldCount = scaleDownTotalOldCount - preDeletingOldRevisionCount } + expectedTotalCount := replicas + res.useSurge + expectedTotalOldCount := partition + res.useSurgeOldRevision + + // scale up + if num := expectedTotalCount - scaleUpTotalCount; num > 0 { + res.scaleUpNum = num + res.scaleUpNumOldRevision = integer.IntMax(expectedTotalOldCount-scaleUpTotalOldCount, 0) - if res.scaleNum > 0 { - res.scaleUpLimit = integer.IntMax(scaleMaxUnavailable-totalUnavailable, 0) - res.scaleUpLimit = integer.IntMin(res.scaleNum, res.scaleUpLimit) + res.scaleUpLimit = integer.IntMin(res.scaleUpNum, integer.IntMax(scaleMaxUnavailable-totalUnavailable, 0)) } - if toDeleteNewRevisionCount > 0 || toDeleteOldRevisionCount > 0 || res.scaleNum < 0 { + // scale down + if num := scaleDownTotalCount - expectedTotalCount; num > 0 { + res.scaleDownNum = num + res.scaleDownNumOldRevision = integer.IntMax(scaleDownTotalOldCount-expectedTotalOldCount, 0) + } + if toDeleteNewRevisionCount > 0 || toDeleteOldRevisionCount > 0 || res.scaleDownNum > 0 { res.deleteReadyLimit = integer.IntMax(maxUnavailable+(len(pods)-replicas)-totalUnavailable, 0) } @@ -245,3 +275,7 @@ func IsPodAvailable(coreControl clonesetcore.Control, pod *v1.Pod, minReadySecon } return coreControl.IsPodUpdateReady(pod, minReadySeconds) } + +func shouldScalingExcludePreparingDelete(cs *appsv1alpha1.CloneSet) bool { + return scalingExcludePreparingDelete || cs.Labels[appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey] == "true" +} diff --git a/pkg/controller/cloneset/sync/cloneset_sync_utils_test.go b/pkg/controller/cloneset/sync/cloneset_sync_utils_test.go index ae5a1f4b6a..df8a0f3214 100644 --- a/pkg/controller/cloneset/sync/cloneset_sync_utils_test.go +++ b/pkg/controller/cloneset/sync/cloneset_sync_utils_test.go @@ -38,6 +38,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { cases := []struct { name string set *appsv1alpha1.CloneSet + setLabels map[string]string pods []*v1.Pod revisionConsistent bool disableFeatureGate bool @@ -53,7 +54,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { name: "increase replicas to 5 (step 1/2)", set: createTestCloneSet(5, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), pods: []*v1.Pod{}, - expectResult: expectationDiffs{scaleNum: 5, scaleUpLimit: 5}, + expectResult: expectationDiffs{scaleUpNum: 5, scaleUpLimit: 5}, }, { name: "increase replicas to 5 (step 2/2)", @@ -88,7 +89,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, }, { name: "specified delete 1 pod (all ready) (step 3/3)", @@ -123,7 +124,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, }, { name: "specified delete 2 pod (all ready) (step 3/6)", @@ -158,7 +159,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, }, { name: "specified delete 2 pod (all ready) (step 6/6)", @@ -182,7 +183,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: -1, deleteReadyLimit: 2}, + expectResult: expectationDiffs{deleteReadyLimit: 2}, }, { name: "specified delete 2 pod and replicas to 4 (step 2/3)", @@ -192,7 +193,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, }, { name: "specified delete 2 pod and replicas to 4 (step 3/3)", @@ -287,7 +288,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, scaleUpLimit: 1}, }, { name: "specified delete with maxSurge (step 2/4)", @@ -337,7 +338,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, updateNum: 2, updateMaxUnavailable: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, updateNum: 2, updateMaxUnavailable: 1, scaleUpLimit: 1}, }, { name: "update in-place partition=3 with maxSurge (step 2/4)", @@ -363,7 +364,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateUpdating, false, false), // new in-place update createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 0}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 0}, }, { name: "update in-place partition=3 with maxSurge (step 4/4)", @@ -376,7 +377,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), // new in-place update createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 1}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 1}, }, { name: "update recreate partition=3 with maxSurge (step 1/7)", @@ -388,7 +389,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, updateNum: 2, updateMaxUnavailable: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, updateNum: 2, updateMaxUnavailable: 1, scaleUpLimit: 1}, }, { name: "update recreate partition=3 with maxSurge (step 2/7)", @@ -426,7 +427,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{useSurge: 1, scaleNum: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{useSurge: 1, scaleUpNum: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, }, { name: "update recreate partition=3 with maxSurge (step 5/7)", @@ -439,7 +440,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation for update }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 0}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 0}, }, { name: "update recreate partition=3 with maxSurge (step 6/7)", @@ -452,7 +453,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), // new creation createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation for update }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 1}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 1}, }, { name: "update recreate partition=3 with maxSurge (step 7/7)", @@ -476,7 +477,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 3, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 3, scaleUpLimit: 1}, }, { name: "update recreate partition=99% with maxUnavailable=3, maxSurge=2 (step 2/3)", @@ -489,7 +490,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 3}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 3}, }, { name: "update recreate partition=99% with maxUnavailable=3, maxSurge=2 (step 3/3)", @@ -513,7 +514,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 2, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 2, scaleUpLimit: 1}, }, { name: "update recreate partition=99% with maxUnavailable=40%, maxSurge=30% (step 2/3)", @@ -526,7 +527,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 2}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 2}, }, { name: "update recreate partition=99% with maxUnavailable=40%, maxSurge=30% (step 3/3)", @@ -550,7 +551,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), }, - expectResult: expectationDiffs{scaleNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, useSurge: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, }, { name: "update recreate partition=99% with maxUnavailable=30%, maxSurge=30% (step 2/3)", @@ -563,7 +564,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(oldRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), // new creation }, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -1, deleteReadyLimit: 1}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 1, deleteReadyLimit: 1}, }, { name: "update recreate partition=99% with maxUnavailable=30%, maxSurge=30% (step 3/3)", @@ -639,7 +640,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, revisionConsistent: true, - expectResult: expectationDiffs{scaleNum: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 1, updateNum: 1, updateMaxUnavailable: 1, scaleUpLimit: 1}, }, { name: "allow to update when fail to scale in normally", @@ -653,7 +654,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), }, revisionConsistent: true, - expectResult: expectationDiffs{scaleNum: -1, scaleNumOldRevision: -2, deleteReadyLimit: 2, updateNum: 1, updateMaxUnavailable: 2}, + expectResult: expectationDiffs{scaleDownNum: 1, scaleDownNumOldRevision: 2, deleteReadyLimit: 2, updateNum: 1, updateMaxUnavailable: 2}, }, { name: "disable rollback feature-gate", @@ -672,7 +673,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { name: "increase replicas 0 to 5 with scale maxUnavailable = 2", set: setScaleStrategy(createTestCloneSet(5, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), intstr.FromInt(2)), pods: []*v1.Pod{}, - expectResult: expectationDiffs{scaleNum: 5, scaleUpLimit: 2}, + expectResult: expectationDiffs{scaleUpNum: 5, scaleUpLimit: 2}, }, { name: "increase replicas 3 to 6 with scale maxUnavailable = 2, not ready pod = 1", @@ -682,7 +683,7 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), }, - expectResult: expectationDiffs{scaleNum: 3, scaleUpLimit: 1}, + expectResult: expectationDiffs{scaleUpNum: 3, scaleUpLimit: 1}, }, { name: "increase replicas 3 to 6 with scale maxUnavailable = 2, not ready pod = 2", @@ -692,7 +693,216 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), }, - expectResult: expectationDiffs{scaleNum: 3, scaleUpLimit: 0}, + expectResult: expectationDiffs{scaleUpNum: 3, scaleUpLimit: 0}, + }, + { + name: "[scalingExcludePreparingDelete=false] specific delete a pod with lifecycle hook (step 1/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, true), + }, + expectResult: expectationDiffs{deleteReadyLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=false] specific delete a pod with lifecycle hook (step 2/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=false] specific delete a pod with lifecycle hook (step 3/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + }, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=false] specific delete a pod with lifecycle hook (step 4/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook (step 1/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, true), + }, + expectResult: expectationDiffs{deleteReadyLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook (step 2/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + }, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook (step 3/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook (step 4/4)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook and then cancel (step 1/5)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, true), + }, + expectResult: expectationDiffs{deleteReadyLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook and then cancel (step 2/5)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + }, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook and then cancel (step 3/5)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook and then cancel (step 4/5)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), // it has been changed to normal by managePreparingDelete + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{scaleDownNum: 1, deleteReadyLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific delete a pod with lifecycle hook and then cancel (step 5/5)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 1/6)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 2/6)", + set: createTestCloneSet(2, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, true), + }, + expectResult: expectationDiffs{deleteReadyLimit: 2}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 3/6)", + set: createTestCloneSet(2, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 4/6)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + }, + expectResult: expectationDiffs{scaleUpNum: 1, scaleUpLimit: 1}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 5/6)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStatePreparingDelete, true, true), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, + }, + { + name: "[scalingExcludePreparingDelete=true] specific scale down with lifecycle hook, then scale up pods (step 6/6)", + set: createTestCloneSet(3, intstr.FromInt(0), intstr.FromInt(1), intstr.FromInt(0)), + setLabels: map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"}, + pods: []*v1.Pod{ + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, true, false), + createTestPod(newRevision, appspub.LifecycleStateNormal, false, false), + }, + expectResult: expectationDiffs{}, }, } @@ -707,6 +917,12 @@ func TestCalculateDiffsWithExpectation(t *testing.T) { if cases[i].revisionConsistent { current = newRevision } + for key, value := range cases[i].setLabels { + if cases[i].set.Labels == nil { + cases[i].set.Labels = map[string]string{} + } + cases[i].set.Labels[key] = value + } res := calculateDiffsWithExpectation(cases[i].set, cases[i].pods, current, newRevision) if !reflect.DeepEqual(res, cases[i].expectResult) { t.Errorf("got %#v, expect %#v", res, cases[i].expectResult) diff --git a/test/e2e/apps/cloneset.go b/test/e2e/apps/cloneset.go index 57af3ece75..5eec81b2a4 100644 --- a/test/e2e/apps/cloneset.go +++ b/test/e2e/apps/cloneset.go @@ -17,13 +17,12 @@ limitations under the License. package apps import ( + "context" "encoding/json" "fmt" "sort" "time" - utilpointer "k8s.io/utils/pointer" - "github.com/onsi/ginkgo" "github.com/onsi/gomega" appspub "github.com/openkruise/kruise/apis/apps/pub" @@ -32,11 +31,15 @@ import ( "github.com/openkruise/kruise/pkg/util" "github.com/openkruise/kruise/test/e2e/framework" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" imageutils "k8s.io/kubernetes/test/utils/image" + utilpointer "k8s.io/utils/pointer" ) var _ = SIGDescribe("CloneSet", func() { @@ -141,6 +144,179 @@ var _ = SIGDescribe("CloneSet", func() { return cs.Status.ReadyReplicas }, 120*time.Second, 3*time.Second).Should(gomega.Equal(int32(3))) }) + + framework.ConformanceIt("specific delete a Pod, when scalingExcludePreparingDelete is disabled", func() { + cs := tester.NewCloneSet("clone-"+randStr, 3, appsv1alpha1.CloneSetUpdateStrategy{}) + cs.Spec.Template.Labels["lifecycle-hook"] = "true" + cs.Spec.Lifecycle = &appspub.Lifecycle{ + PreDelete: &appspub.LifecycleHook{ + LabelsHandler: map[string]string{"lifecycle-hook": "true"}, + }, + } + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(3))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, 3*time.Second).Should(gomega.Equal(int32(3))) + + oldPods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(oldPods).To(gomega.HaveLen(int(cs.Status.Replicas))) + + specifiedPodName := oldPods[0].Name + ginkgo.By(fmt.Sprintf("Patch Pod %s with specified-delete label", specifiedPodName)) + patchBody := []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"true"}}}`, appsv1alpha1.SpecifiedDeleteKey)) + _, err = c.CoreV1().Pods(cs.Namespace).Patch(context.TODO(), specifiedPodName, types.StrategicMergePatchType, patchBody, metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait specified pod becoming PreparingDelete") + gomega.Eventually(func() appspub.LifecycleStateType { + pod, err := c.CoreV1().Pods(cs.Namespace).Get(context.TODO(), specifiedPodName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return appspub.LifecycleStateType(pod.Labels[appspub.LifecycleStateKey]) + }, 10*time.Second, time.Second).Should(gomega.Equal(appspub.LifecycleStatePreparingDelete)) + + ginkgo.By("Should not scale up") + gomega.Consistently(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 5*time.Second, time.Second).Should(gomega.Equal(int32(3))) + + newPods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(util.GetPodNames(newPods).List()).Should(gomega.Equal(util.GetPodNames(newPods).List())) + + ginkgo.By("Remove lifecycle hook label and wait it to be deleted") + patchBody = []byte(`{"metadata":{"labels":{"lifecycle-hook":null}}}`) + _, err = c.CoreV1().Pods(cs.Namespace).Patch(context.TODO(), specifiedPodName, types.StrategicMergePatchType, patchBody, metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() *v1.Pod { + pod, err := c.CoreV1().Pods(cs.Namespace).Get(context.TODO(), specifiedPodName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + return pod + }, 30*time.Second, time.Second).Should(gomega.BeNil()) + + ginkgo.By("Wait new Pod created and two old Pods should be still running") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, 3*time.Second).Should(gomega.Equal(int32(3))) + + newPods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(newPods).To(gomega.HaveLen(int(cs.Status.Replicas))) + + keepOldPods := util.GetPodNames(newPods).Intersection(util.GetPodNames(oldPods)).List() + gomega.Expect(keepOldPods).To(gomega.HaveLen(2)) + }) + + framework.ConformanceIt("specific scale down with lifecycle and then scale up, when scalingExcludePreparingDelete is enabled", func() { + cs := tester.NewCloneSet("clone-"+randStr, 3, appsv1alpha1.CloneSetUpdateStrategy{}) + cs.Labels = map[string]string{appsv1alpha1.CloneSetScalingExcludePreparingDeleteKey: "true"} + cs.Spec.Template.Labels["lifecycle-hook"] = "true" + cs.Spec.Lifecycle = &appspub.Lifecycle{ + PreDelete: &appspub.LifecycleHook{ + LabelsHandler: map[string]string{"lifecycle-hook": "true"}, + }, + } + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(3))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, 3*time.Second).Should(gomega.Equal(int32(3))) + + oldPods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(oldPods).To(gomega.HaveLen(int(cs.Status.Replicas))) + + specifiedPodName := oldPods[0].Name + ginkgo.By(fmt.Sprintf("Scale down replicas=2 with specified Pod %s", specifiedPodName)) + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32(2) + cs.Spec.ScaleStrategy.PodsToDelete = []string{specifiedPodName} + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait specified pod becoming PreparingDelete") + gomega.Eventually(func() appspub.LifecycleStateType { + pod, err := c.CoreV1().Pods(cs.Namespace).Get(context.TODO(), specifiedPodName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return appspub.LifecycleStateType(pod.Labels[appspub.LifecycleStateKey]) + }, 10*time.Second, time.Second).Should(gomega.Equal(appspub.LifecycleStatePreparingDelete)) + + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cs.Status.Replicas).To(gomega.Equal(int32(3))) + + ginkgo.By("Scale up to 3 again and wait status.replicas to be 4") + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32(3) + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 10*time.Second, 3*time.Second).Should(gomega.Equal(int32(4))) + + ginkgo.By("Remove lifecycle hook label and wait it to be deleted") + patchBody := []byte(`{"metadata":{"labels":{"lifecycle-hook":null}}}`) + _, err = c.CoreV1().Pods(cs.Namespace).Patch(context.TODO(), specifiedPodName, types.StrategicMergePatchType, patchBody, metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() *v1.Pod { + pod, err := c.CoreV1().Pods(cs.Namespace).Get(context.TODO(), specifiedPodName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + return pod + }, 30*time.Second, time.Second).Should(gomega.BeNil()) + + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, 3*time.Second).Should(gomega.Equal(int32(3))) + + newPods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(newPods).To(gomega.HaveLen(int(cs.Status.Replicas))) + + keepOldPods := util.GetPodNames(newPods).Intersection(util.GetPodNames(oldPods)).List() + gomega.Expect(keepOldPods).To(gomega.HaveLen(2)) + }) }) framework.KruiseDescribe("CloneSet Updating", func() {