From 8a1dad75e32cc605f725661da800b9acde325f25 Mon Sep 17 00:00:00 2001 From: linsongzheng Date: Fri, 25 Nov 2022 20:59:28 +0800 Subject: [PATCH 1/5] Update stateful_set_utils.go Fix all unavailable pod update new version fail Signed-off-by: linsongzheng --- pkg/controller/statefulset/stateful_set_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index fd016f5c60..f1a18456d7 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -70,7 +70,7 @@ func getParentName(pod *v1.Pod) string { return parent } -// getOrdinal gets pod's ordinal. If pod has no ordinal, -1 is returned. +// getOrdinal gets pod's ordinal. If pod has no ordinal, -1 is returned. func getOrdinal(pod *v1.Pod) int { _, ordinal := getParentNameAndOrdinal(pod) return ordinal @@ -687,5 +687,5 @@ func decreaseAndCheckMaxUnavailable(maxUnavailable *int) bool { } val := *maxUnavailable - 1 *maxUnavailable = val - return val <= 0 + return val < 0 } From d03e83e619a701c155663b9306d94087baf6cc68 Mon Sep 17 00:00:00 2001 From: linsongzheng Date: Fri, 2 Dec 2022 16:40:32 +0800 Subject: [PATCH 2/5] Fix will be create one more pod Signed-off-by: linsongzheng --- pkg/controller/statefulset/stateful_set_control.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index ec93929bf1..71ce98533e 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -527,6 +527,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } } + // if we find no more Pods can be created, no more work can be done this round + if decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + break + } + lifecycle.SetPodLifecycle(appspub.LifecycleStateNormal)(replicas[i]) if err := ssc.podControl.CreateStatefulPod(set, replicas[i]); err != nil { msg := fmt.Sprintf("StatefulPodControl failed to create Pod error: %s", err) @@ -542,7 +547,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( status.UpdatedReplicas++ } // if the set does not allow bursting, return immediately - if monotonic || decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + if monotonic { return &status, nil } // pod created, no more work possible for this round From 4d8b58f7bafe82329590d9fdc0016b15d64eded9 Mon Sep 17 00:00:00 2001 From: linsongzheng Date: Mon, 12 Dec 2022 10:21:41 +0800 Subject: [PATCH 3/5] add scaleStrategy e2e test Signed-off-by: linsongzheng --- .../statefulset/stateful_set_control.go | 19 +++-- .../statefulset/stateful_set_utils.go | 2 +- test/e2e/apps/statefulset.go | 78 +++++++++++++++++++ 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 71ce98533e..13aecf905d 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -527,11 +527,6 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } } - // if we find no more Pods can be created, no more work can be done this round - if decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { - break - } - lifecycle.SetPodLifecycle(appspub.LifecycleStateNormal)(replicas[i]) if err := ssc.podControl.CreateStatefulPod(set, replicas[i]); err != nil { msg := fmt.Sprintf("StatefulPodControl failed to create Pod error: %s", err) @@ -549,19 +544,23 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // if the set does not allow bursting, return immediately if monotonic { return &status, nil + } else if decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + break } // pod created, no more work possible for this round continue } // If we find a Pod that is currently terminating, we must wait until graceful deletion // completes before we continue to make progress. - if isTerminating(replicas[i]) && (monotonic || decreaseAndCheckMaxUnavailable(scaleMaxUnavailable)) { + if isTerminating(replicas[i]) && monotonic { klog.V(4).Infof( "StatefulSet %s/%s is waiting for Pod %s to Terminate", set.Namespace, set.Name, replicas[i].Name) return &status, nil + } else if isTerminating(replicas[i]) && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + break } // Update InPlaceUpdateReady condition for pod if res := ssc.inplaceControl.Refresh(replicas[i], nil); res.RefreshErr != nil { @@ -576,7 +575,7 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // ordinal, are Running and Available. if monotonic || scaleMaxUnavailable != nil { isAvailable, waitTime := isRunningAndAvailable(replicas[i], minReadySeconds) - if !isAvailable && (monotonic || decreaseAndCheckMaxUnavailable(scaleMaxUnavailable)) { + if !isAvailable && monotonic { if waitTime > 0 { // make sure we check later durationStore.Push(getStatefulSetKey(set), waitTime) @@ -596,6 +595,12 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( replicas[i].Name) } return &status, nil + } else if !isAvailable && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + if waitTime > 0 { + // make sure we check later + durationStore.Push(getStatefulSetKey(set), waitTime) + } + break } } // Enforce the StatefulSet invariants diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index f1a18456d7..d8ea3d97c8 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -687,5 +687,5 @@ func decreaseAndCheckMaxUnavailable(maxUnavailable *int) bool { } val := *maxUnavailable - 1 *maxUnavailable = val - return val < 0 + return val <= 0 } diff --git a/test/e2e/apps/statefulset.go b/test/e2e/apps/statefulset.go index ac3bf44fec..37b91e2ae5 100644 --- a/test/e2e/apps/statefulset.go +++ b/test/e2e/apps/statefulset.go @@ -31,11 +31,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" klabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" watchtools "k8s.io/client-go/tools/watch" imageutils "k8s.io/kubernetes/test/utils/image" + "k8s.io/utils/pointer" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -1031,6 +1033,82 @@ var _ = SIGDescribe("StatefulSet", func() { } framework.ExpectEqual(*(ss.Spec.Replicas), int32(2)) }) + + /* + Testname: StatefulSet, ScaleStrategy + Description: StatefulSet resource MUST support the MaxUnavailable ScaleStrategy for scaling. + It only affects when create new pod, terminating pod and unavailable pod at the Parallel PodManagementPolicy. + */ + framework.ConformanceIt("Should can update pods when the statefulset scale strategy is set", func() { + ginkgo.By("Creating statefulset " + ssName + " in namespace " + ns) + maxUnavailable := intstr.FromInt(2) + ss := framework.NewStatefulSet(ssName, ns, headlessSvcName, 3, nil, nil, labels) + ss.Spec.Template.Spec.Containers[0].Name = "busybox" + ss.Spec.Template.Spec.Containers[0].Image = BusyboxImage + ss.Spec.Template.Spec.Containers[0].Command = []string{"sleep", "3600"} + ss.Spec.PodManagementPolicy = apps.ParallelPodManagement + ss.Spec.Template.Spec.RestartPolicy = v1.RestartPolicyAlways + ss.Spec.UpdateStrategy.RollingUpdate = &appsv1beta1.RollingUpdateStatefulSetStrategy{ + MinReadySeconds: pointer.Int32(3), + PodUpdatePolicy: appsv1beta1.InPlaceIfPossiblePodUpdateStrategyType, + } + ss.Spec.ScaleStrategy = &appsv1beta1.StatefulSetScaleStrategy{MaxUnavailable: &maxUnavailable} + ss.Spec.Template.Spec.ReadinessGates = append(ss.Spec.Template.Spec.ReadinessGates, v1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady}) + sst := framework.NewStatefulSetTester(c, kc) + // sst.SetHTTPProbe(ss) + ss, err := kc.AppsV1beta1().StatefulSets(ns).Create(context.TODO(), ss, metav1.CreateOptions{}) + framework.ExpectNoError(err) + sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss) + + ginkgo.By("Scaling up stateful set " + ssName + " to 10 replicas and check create new pod equal MaxUnavailable") + sst.UpdateReplicas(ss, 10) + sst.ConfirmStatefulPodCount(5, ss, time.Second, false) + sst.WaitForRunningAndReady(10, ss) + + ginkgo.By("Confirming that stateful can update all pods to be unhealthy") + maxUnavailable = intstr.FromString("100%") + ss, err = framework.UpdateStatefulSetWithRetries(kc, ns, ss.Name, func(update *appsv1beta1.StatefulSet) { + update.Spec.ScaleStrategy.MaxUnavailable = &maxUnavailable + update.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &intstr.IntOrString{Type: intstr.String, StrVal: "100%"} + update.Spec.Template.Spec.Containers[0].Command = []string{} + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + sst.WaitForRunningAndNotReady(10, ss) + sst.WaitForStatusReadyReplicas(ss, 0) + ss = sst.WaitForStatus(ss) + + ginkgo.By("Confirming that stateful can update all pods if any stateful pod is unhealthy") + + ss, err = framework.UpdateStatefulSetWithRetries(kc, ns, ss.Name, func(update *appsv1beta1.StatefulSet) { + update.Spec.Template.Labels["test-update"] = "yes" + update.Spec.Template.Spec.Containers[0].Command = []string{"sleep", "180"} + }) + sst.WaitForRunningAndReady(10, ss) + sst.WaitForStatusReadyReplicas(ss, 10) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var pods *v1.PodList + sst.WaitForState(ss, func(set *appsv1beta1.StatefulSet, pl *v1.PodList) (bool, error) { + ss = set + pods = pl + sst.SortStatefulPods(pods) + for i := range pods.Items { + if pods.Items[i].Labels[apps.StatefulSetRevisionLabel] != set.Status.UpdateRevision { + framework.Logf("Waiting for Pod %s/%s to have revision %s update revision %s", + pods.Items[i].Namespace, + pods.Items[i].Name, + set.Status.UpdateRevision, + pods.Items[i].Labels[apps.StatefulSetRevisionLabel]) + return false, nil + } + } + return true, nil + }) + + ginkgo.By("Confirming Pods were updated successful") + for i := range pods.Items { + gomega.Expect(pods.Items[i].Labels["test-update"]).To(gomega.Equal("yes")) + } + }) }) //ginkgo.Describe("Deploy clustered applications [Feature:StatefulSet] [Slow]", func() { From d931cdd6c37bb93005566a723df1cb1c94b92201 Mon Sep 17 00:00:00 2001 From: linsongzheng Date: Wed, 14 Dec 2022 09:51:40 +0800 Subject: [PATCH 4/5] add log Signed-off-by: linsongzheng --- pkg/controller/statefulset/stateful_set_control.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 13aecf905d..fa68b886d4 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -560,6 +560,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( replicas[i].Name) return &status, nil } else if isTerminating(replicas[i]) && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + klog.V(4).Infof( + "StatefulSet %s/%s Pod %s is Terminating, and break pods scale", + set.Namespace, + set.Name, + replicas[i].Name) break } // Update InPlaceUpdateReady condition for pod @@ -596,6 +601,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( } return &status, nil } else if !isAvailable && decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + klog.V(4).Infof( + "StatefulSet %s/%s Pod %s is unavailable, and break pods scale", + set.Namespace, + set.Name, + replicas[i].Name) if waitTime > 0 { // make sure we check later durationStore.Push(getStatefulSetKey(set), waitTime) From ee7f0d64230a876abfc75061a9880c03fcc9463d Mon Sep 17 00:00:00 2001 From: linsongzheng Date: Wed, 14 Dec 2022 09:53:04 +0800 Subject: [PATCH 5/5] add log Signed-off-by: linsongzheng --- pkg/controller/statefulset/stateful_set_control.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index fa68b886d4..275cb0d047 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -545,6 +545,11 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( if monotonic { return &status, nil } else if decreaseAndCheckMaxUnavailable(scaleMaxUnavailable) { + klog.V(4).Infof( + "StatefulSet %s/%s Pod %s is Creating, and break pods scale", + set.Namespace, + set.Name, + replicas[i].Name) break } // pod created, no more work possible for this round