add start ordinal and e2e case (#1643)

Signed-off-by: Abner-1 <yuanyuxing.yyx@alibaba-inc.com>

apis/apps/v1beta1/statefulset_types.go

@@ -17,11 +17,12 @@ limitations under the License.
 package v1beta1
 
 import (
-	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 )
 
 const (
@@ -143,6 +144,21 @@ type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
 	WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
 }
 
+// StatefulSetOrdinals describes the policy used for replica ordinal assignment
+// in this StatefulSet.
+type StatefulSetOrdinals struct {
+	// start is the number representing the first replica's index. It may be used
+	// to number replicas from an alternate index (eg: 1-indexed) over the default
+	// 0-indexed names, or to orchestrate progressive movement of replicas from
+	// one StatefulSet to another.
+	// If set, replica indices will be in the range:
+	//   [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+	// If unset, defaults to 0. Replica indices will be in the range:
+	//   [0, .spec.replicas).
+	// +optional
+	Start int32 `json:"start" protobuf:"varint,1,opt,name=start"`
+}
+
 // StatefulSetSpec defines the desired state of StatefulSet
 type StatefulSetSpec struct {
 	// replicas is the desired number of replicas of the given Template.
@@ -228,6 +244,14 @@ type StatefulSetSpec struct {
 	// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
 	// +optional
 	PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
+
+	// ordinals controls the numbering of replica indices in a StatefulSet. The
+	// default ordinals behavior assigns a "0" index to the first replica and
+	// increments the index by one for each additional replica requested. Using
+	// the ordinals field requires the StatefulSetStartOrdinal feature gate to be
+	// enabled, which is beta.
+	// +optional
+	Ordinals *StatefulSetOrdinals `json:"ordinals,omitempty"`
 }
 
 // StatefulSetScaleStrategy defines strategies for pods scale.

config/crd/bases/apps.kruise.io_statefulsets.yaml

@@ -580,6 +580,27 @@ spec:
                         type: boolean
                     type: object
                 type: object
+              ordinals:
+                description: |-
+                  ordinals controls the numbering of replica indices in a StatefulSet. The
+                  default ordinals behavior assigns a "0" index to the first replica and
+                  increments the index by one for each additional replica requested. Using
+                  the ordinals field requires the StatefulSetStartOrdinal feature gate to be
+                  enabled, which is beta.
+                properties:
+                  start:
+                    description: |-
+                      start is the number representing the first replica's index. It may be used
+                      to number replicas from an alternate index (eg: 1-indexed) over the default
+                      0-indexed names, or to orchestrate progressive movement of replicas from
+                      one StatefulSet to another.
+                      If set, replica indices will be in the range:
+                        [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+                      If unset, defaults to 0. Replica indices will be in the range:
+                        [0, .spec.replicas).
+                    format: int32
+                    type: integer
+                type: object
               persistentVolumeClaimRetentionPolicy:
                 description: |-
                   PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from

config/crd/bases/apps.kruise.io_uniteddeployments.yaml

@@ -203,6 +203,27 @@ spec:
                                     type: boolean
                                 type: object
                             type: object
+                          ordinals:
+                            description: |-
+                              ordinals controls the numbering of replica indices in a StatefulSet. The
+                              default ordinals behavior assigns a "0" index to the first replica and
+                              increments the index by one for each additional replica requested. Using
+                              the ordinals field requires the StatefulSetStartOrdinal feature gate to be
+                              enabled, which is beta.
+                            properties:
+                              start:
+                                description: |-
+                                  start is the number representing the first replica's index. It may be used
+                                  to number replicas from an alternate index (eg: 1-indexed) over the default
+                                  0-indexed names, or to orchestrate progressive movement of replicas from
+                                  one StatefulSet to another.
+                                  If set, replica indices will be in the range:
+                                    [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+                                  If unset, defaults to 0. Replica indices will be in the range:
+                                    [0, .spec.replicas).
+                                format: int32
+                                type: integer
+                            type: object
                           persistentVolumeClaimRetentionPolicy:
                             description: |-
                               PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from

go.mod

@@ -104,7 +104,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.6.0
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect

pkg/controller/statefulset/stateful_pod_control.go

@@ -315,6 +315,22 @@ func (spc *StatefulPodControl) recordClaimEvent(verb string, set *appsv1beta1.St
 	}
 }
 
+// createMissingPersistentVolumeClaims creates all of the required PersistentVolumeClaims for pod, and updates its retention policy
+func (spc *StatefulPodControl) createMissingPersistentVolumeClaims(ctx context.Context, set *appsv1beta1.StatefulSet, pod *v1.Pod) error {
+	if err := spc.createPersistentVolumeClaims(set, pod); err != nil {
+		return err
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
+		// Set PVC policy as much as is possible at this point.
+		if err := spc.UpdatePodClaimForRetentionPolicy(set, pod); err != nil {
+			spc.recordPodEvent("update", set, pod, err)
+			return err
+		}
+	}
+	return nil
+}
+
 // createPersistentVolumeClaims creates all of the required PersistentVolumeClaims for pod, which must be a member of
 // set. If all of the claims for Pod are successfully created, the returned error is nil. If creation fails, this method
 // may be called again until no error is returned, indicating the PersistentVolumeClaims for pod are consistent with

pkg/controller/statefulset/stateful_pod_control_test.go

@@ -27,10 +27,6 @@ import (
 	"testing"
 	"time"
 
-	appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
-	"github.com/openkruise/kruise/pkg/features"
-	"github.com/openkruise/kruise/pkg/util"
-	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -46,6 +42,11 @@ import (
 	_ "k8s.io/kubernetes/pkg/apis/apps/install"
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	utilpointer "k8s.io/utils/pointer"
+
+	appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
+	"github.com/openkruise/kruise/pkg/features"
+	"github.com/openkruise/kruise/pkg/util"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 )
 
 func TestStatefulPodControlCreatesPods(t *testing.T) {
@@ -872,11 +873,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 				return set
 			},
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(set)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(set)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 2, 3, 4}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -909,11 +910,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 				return set
 			},
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(set)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(set)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 2, 3, 4}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -948,11 +949,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
 				setClone := set.DeepCopy()
 				setClone.Spec.Replicas = utilpointer.Int32(5)
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(setClone)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 2, 3, 4}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -996,11 +997,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
 				setClone := set.DeepCopy()
 				setClone.Spec.Replicas = utilpointer.Int32(5)
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(setClone)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 2, 3, 4}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -1043,11 +1044,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 				return set
 			},
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(set)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(set)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 3, 5, 6}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -1081,11 +1082,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 				return set
 			},
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(set)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(set)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 3, 5, 6}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -1121,11 +1122,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
 				setClone := set.DeepCopy()
 				setClone.Spec.Replicas = utilpointer.Int32(5)
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(setClone)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 3, 5, 6}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}
@@ -1170,11 +1171,11 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
 			getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
 				setClone := set.DeepCopy()
 				setClone.Spec.Replicas = utilpointer.Int32(5)
-				replicaCount, reserveOrdinals := getStatefulSetReplicasRange(setClone)
+				startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone)
 				pods := make([]*v1.Pod, 0)
 				expectIndex := []int{0, 1, 3, 5, 6}
 				currentIndex := make([]int, 0)
-				for i := 0; i < replicaCount; i++ {
+				for i := startOrdinal; i < endOrdinal; i++ {
 					if reserveOrdinals.Has(i) {
 						continue
 					}