add postCreateHook for lifecycle

Signed-off-by: mingzhou.swx <mingzhou.swx@alibaba-inc.com>

apis/apps/pub/lifecycle.go

@@ -20,6 +20,7 @@ const (
 	LifecycleStateKey     = "lifecycle.apps.kruise.io/state"
 	LifecycleTimestampKey = "lifecycle.apps.kruise.io/timestamp"
 
+	LifecycleStateCreating        LifecycleStateType = "Creating"
 	LifecycleStateNormal          LifecycleStateType = "Normal"
 	LifecycleStatePreparingUpdate LifecycleStateType = "PreparingUpdate"
 	LifecycleStateUpdating        LifecycleStateType = "Updating"
@@ -35,6 +36,8 @@ type Lifecycle struct {
 	PreDelete *LifecycleHook `json:"preDelete,omitempty"`
 	// InPlaceUpdate is the hook before Pod to update and after Pod has been updated.
 	InPlaceUpdate *LifecycleHook `json:"inPlaceUpdate,omitempty"`
+	// PostCreated is the hook after Pod to be created and ready to be Normal.
+	PostCreated *LifecycleHook `json:"postCreated,omitempty"`
 }
 
 type LifecycleHook struct {

config/crd/bases/apps.kruise.io_clonesets.yaml

@@ -105,6 +105,26 @@ spec:
                           to false.'
                         type: boolean
                     type: object
+                  postCreated:
+                    description: PostCreated is the hook after Pod to be created and
+                      ready to be Normal.
+                    properties:
+                      finalizersHandler:
+                        items:
+                          type: string
+                        type: array
+                      labelsHandler:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      markPodNotReady:
+                        description: 'MarkPodNotReady = true means: - Pod will be
+                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          - Pod will be restored to ''Ready'' at Updated state if
+                          it was set to ''NotReady'' at preparingUpdate state. Default
+                          to false.'
+                        type: boolean
+                    type: object
                   preDelete:
                     description: PreDelete is the hook before Pod to be deleted.
                     properties:

config/crd/bases/apps.kruise.io_daemonsets.yaml

@@ -109,6 +109,26 @@ spec:
                           to false.'
                         type: boolean
                     type: object
+                  postCreated:
+                    description: PostCreated is the hook after Pod to be created and
+                      ready to be Normal.
+                    properties:
+                      finalizersHandler:
+                        items:
+                          type: string
+                        type: array
+                      labelsHandler:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      markPodNotReady:
+                        description: 'MarkPodNotReady = true means: - Pod will be
+                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          - Pod will be restored to ''Ready'' at Updated state if
+                          it was set to ''NotReady'' at preparingUpdate state. Default
+                          to false.'
+                        type: boolean
+                    type: object
                   preDelete:
                     description: PreDelete is the hook before Pod to be deleted.
                     properties:

config/crd/bases/apps.kruise.io_statefulsets.yaml

@@ -522,6 +522,26 @@ spec:
                           to false.'
                         type: boolean
                     type: object
+                  postCreated:
+                    description: PostCreated is the hook after Pod to be created and
+                      ready to be Normal.
+                    properties:
+                      finalizersHandler:
+                        items:
+                          type: string
+                        type: array
+                      labelsHandler:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      markPodNotReady:
+                        description: 'MarkPodNotReady = true means: - Pod will be
+                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          - Pod will be restored to ''Ready'' at Updated state if
+                          it was set to ''NotReady'' at preparingUpdate state. Default
+                          to false.'
+                        type: boolean
+                    type: object
                   preDelete:
                     description: PreDelete is the hook before Pod to be deleted.
                     properties:

config/crd/bases/apps.kruise.io_uniteddeployments.yaml

@@ -153,6 +153,26 @@ spec:
                                       at preparingUpdate state. Default to false.'
                                     type: boolean
                                 type: object
+                              postCreated:
+                                description: PostCreated is the hook after Pod to
+                                  be created and ready to be Normal.
+                                properties:
+                                  finalizersHandler:
+                                    items:
+                                      type: string
+                                    type: array
+                                  labelsHandler:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
+                                  markPodNotReady:
+                                    description: 'MarkPodNotReady = true means: -
+                                      Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate
+                                      state. - Pod will be restored to ''Ready'' at
+                                      Updated state if it was set to ''NotReady''
+                                      at preparingUpdate state. Default to false.'
+                                    type: boolean
+                                type: object
                               preDelete:
                                 description: PreDelete is the hook before Pod to be
                                   deleted.
@@ -568,6 +588,26 @@ spec:
                                       at preparingUpdate state. Default to false.'
                                     type: boolean
                                 type: object
+                              postCreated:
+                                description: PostCreated is the hook after Pod to
+                                  be created and ready to be Normal.
+                                properties:
+                                  finalizersHandler:
+                                    items:
+                                      type: string
+                                    type: array
+                                  labelsHandler:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
+                                  markPodNotReady:
+                                    description: 'MarkPodNotReady = true means: -
+                                      Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate
+                                      state. - Pod will be restored to ''Ready'' at
+                                      Updated state if it was set to ''NotReady''
+                                      at preparingUpdate state. Default to false.'
+                                    type: boolean
+                                type: object
                               preDelete:
                                 description: PreDelete is the hook before Pod to be
                                   deleted.

pkg/controller/cloneset/sync/cloneset_scale.go

@@ -172,9 +172,9 @@ func (r *realControl) managePreparingDelete(cs *appsv1alpha1.CloneSet, pods, pod
 			continue
 		}
 
-		klog.V(3).Infof("CloneSet %s patch pod %s lifecycle from PreparingDelete to Normal",
+		klog.V(3).Infof("CloneSet %s patch pod %s lifecycle from PreparingDelete to Creating",
 			clonesetutils.GetControllerKey(cs), pod.Name)
-		if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateNormal, false); err != nil {
+		if updated, gotPod, err := r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateCreating, false); err != nil {
 			return modified, err
 		} else if updated {
 			modified = true
@@ -214,7 +214,7 @@ func (r *realControl) createPods(
 		if clonesetutils.EqualToRevisionHash("", pod, currentRevision) {
 			cs = currentCS
 		}
-		lifecycle.SetPodLifecycle(appspub.LifecycleStateNormal)(pod)
+		lifecycle.SetPodLifecycle(appspub.LifecycleStateCreating)(pod)
 
 		var createErr error
 		if createErr = r.createOnePod(cs, pod, existingPVCNames); createErr != nil {

pkg/controller/cloneset/sync/cloneset_update.go

@@ -168,6 +168,13 @@ func (c *realControl) refreshPodState(cs *appsv1alpha1.CloneSet, coreControl clo
 
 	var state appspub.LifecycleStateType
 	switch lifecycle.GetPodLifecycleState(pod) {
+	case appspub.LifecycleStateCreating:
+		if cs.Spec.Lifecycle == nil ||
+			cs.Spec.Lifecycle.PostCreated == nil ||
+			lifecycle.IsPodAllHooked(cs.Spec.Lifecycle.PostCreated, pod) {
+			state = appspub.LifecycleStateNormal
+		}
+
 	case appspub.LifecycleStateUpdating:
 		if opts.CheckPodUpdateCompleted(pod) == nil {
 			if cs.Spec.Lifecycle != nil && !lifecycle.IsPodAllHooked(cs.Spec.Lifecycle.InPlaceUpdate, pod) {

pkg/controller/podreadiness/pod_readiness_controller.go

@@ -18,6 +18,11 @@ package podreadiness
 
 import (
 	"context"
+	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
+	lifecycleutil "github.com/openkruise/kruise/pkg/util/lifecycle"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"time"
 
 	appspub "github.com/openkruise/kruise/apis/apps/pub"
@@ -38,6 +43,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 )
 
+const (
+	CloneSetKind            = "CloneSet"
+	AdvancedStatefulSetKind = "StatefulSet"
+	AdvancedDaemonSetKind   = "DaemonSet"
+)
+
 var (
 	concurrentReconciles = 3
 )
@@ -48,8 +59,10 @@ func Add(mgr manager.Manager) error {
 
 // newReconciler returns a new reconcile.Reconciler
 func newReconciler(mgr manager.Manager) *ReconcilePodReadiness {
+	cli := utilclient.NewClientFromManager(mgr, "pod-readiness-controller")
 	return &ReconcilePodReadiness{
-		Client: utilclient.NewClientFromManager(mgr, "pod-readiness-controller"),
+		Client:           cli,
+		lifecycleControl: lifecycleutil.New(cli),
 	}
 }
 
@@ -89,6 +102,7 @@ var _ reconcile.Reconciler = &ReconcilePodReadiness{}
 // ReconcilePodReadiness reconciles a Pod object
 type ReconcilePodReadiness struct {
 	client.Client
+	lifecycleControl lifecycleutil.Interface
 }
 
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
@@ -127,6 +141,19 @@ func (r *ReconcilePodReadiness) Reconcile(_ context.Context, request reconcile.R
 			return nil
 		}
 
+		lifecycle, err := r.getLifecycleHook(pod)
+		if err != nil {
+			return err
+		}
+
+		if lifecycle != nil &&
+			lifecycle.PostCreated != nil &&
+			lifecycle.PostCreated.MarkPodNotReady &&
+			!lifecycleutil.IsPodAllHooked(lifecycle.PostCreated, pod) {
+			_, _, err = r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateCreating, true)
+			return err
+		}
+
 		pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{
 			Type:               appspub.KruisePodReadyConditionType,
 			Status:             v1.ConditionTrue,
@@ -136,3 +163,45 @@ func (r *ReconcilePodReadiness) Reconcile(_ context.Context, request reconcile.R
 	})
 	return reconcile.Result{}, err
 }
+
+func (r *ReconcilePodReadiness) getLifecycleHook(pod *v1.Pod) (*appspub.Lifecycle, error) {
+	if pod.Labels[appspub.LifecycleStateKey] != string(appspub.LifecycleStateCreating) {
+		return nil, nil
+	}
+
+	owner := metav1.GetControllerOfNoCopy(pod)
+	if owner == nil {
+		return nil, nil
+	}
+
+	ownerGVK := schema.FromAPIVersionAndKind(owner.APIVersion, owner.Kind)
+	if ownerGVK.Group != appsv1alpha1.GroupVersion.Group {
+		return nil, nil
+	}
+	ownerKey := types.NamespacedName{Namespace: pod.Namespace, Name: owner.Name}
+
+	switch ownerGVK.Kind {
+	case CloneSetKind:
+		clone := &appsv1alpha1.CloneSet{}
+		if err := r.Get(context.TODO(), ownerKey, clone); err != nil {
+			return nil, err
+		}
+		return clone.Spec.Lifecycle, nil
+
+	case AdvancedStatefulSetKind:
+		asts := &appsv1beta1.StatefulSet{}
+		if err := r.Get(context.TODO(), ownerKey, asts); err != nil {
+			return nil, err
+		}
+		return asts.Spec.Lifecycle, nil
+
+	case AdvancedDaemonSetKind:
+		ads := &appsv1alpha1.DaemonSet{}
+		if err := r.Get(context.TODO(), ownerKey, ads); err != nil {
+			return nil, err
+		}
+		return ads.Spec.Lifecycle, nil
+	}
+
+	return nil, nil
+}

pkg/util/lifecycle/lifecycle_utils.go

@@ -37,6 +37,7 @@ const (
 	// these keys for MarkPodNotReady Policy of pod lifecycle
 	preparingDeleteHookKey = "preDeleteHook"
 	preparingUpdateHookKey = "preUpdateHook"
+	postCreateHookKey      = "postCreateHook"
 )
 
 func newPodReadinessControl(adp podadapter.Adapter) podreadiness.Interface {
@@ -111,6 +112,10 @@ func SetPodLifecycle(state appspub.LifecycleStateType) func(*v1.Pod) {
 
 func (c *realControl) executePodNotReadyPolicy(pod *v1.Pod, state appspub.LifecycleStateType) (err error) {
 	switch state {
+	case appspub.LifecycleStateCreating:
+		err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(postCreateHookKey))
+	case appspub.LifecycleStateNormal:
+		err = c.podReadinessControl.RemoveNotReadyKey(pod, getReadinessMessage(postCreateHookKey))
 	case appspub.LifecycleStatePreparingDelete:
 		err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(preparingDeleteHookKey))
 	case appspub.LifecycleStatePreparingUpdate:

pkg/util/podreadiness/pod_readiness_utils.go

@@ -46,7 +46,7 @@ func addNotReadyKey(adp podadapter.Adapter, pod *v1.Pod, msg Message, condType v
 		if condition == nil {
 			_, messages := addMessage("", msg)
 			newPod.Status.Conditions = append(newPod.Status.Conditions, v1.PodCondition{
-				Type:               appspub.KruisePodReadyConditionType,
+				Type:               condType,
 				Message:            messages.dump(),
 				LastTransitionTime: metav1.Now(),
 			})

pkg/webhook/statefulset/validating/statefulset_validation.go

@@ -321,6 +321,7 @@ func ValidateStatefulSetUpdate(statefulSet, oldStatefulSet *appsv1beta1.Stateful
 	restoreReserveOrdinals := statefulSet.Spec.ReserveOrdinals
 	statefulSet.Spec.ReserveOrdinals = oldStatefulSet.Spec.ReserveOrdinals
 	statefulSet.Spec.Lifecycle = oldStatefulSet.Spec.Lifecycle
+	statefulSet.Spec.VolumeClaimTemplates = oldStatefulSet.Spec.VolumeClaimTemplates
 	statefulSet.Spec.RevisionHistoryLimit = oldStatefulSet.Spec.RevisionHistoryLimit
 
 	if !apiequality.Semantic.DeepEqual(statefulSet.Spec, oldStatefulSet.Spec) {