fix, podopslifecycle use available-conditions (#23)

apis/apps/v1alpha1/types.go

@@ -15,3 +15,8 @@ limitations under the License.
 */
 
 package v1alpha1
+
+// +kubebuilder:object:generate=false
+type PodAvailableConditions struct {
+	ExpectedFinalizers []string `json:"expectedFinalizers,omitempty"` // indicate the expected finalizers of a pod
+}

apis/apps/v1alpha1/well_known_annotations.go

@@ -17,9 +17,5 @@ limitations under the License.
 package v1alpha1
 
 const (
-	// --- Begin: Annotations for RuleSet ---
-
-	AnnotationPodSkipRuleConditions = "apps.kafed.io/skip-rule-conditions"
-
-	// --- End: Annotations for RuleSet ---
+	PodAvailableConditionsAnnotation = "cafed.kusionstack.io/available-conditions" // indicate the available conditions of a pod
 )

apis/apps/v1alpha1/well_known_labels.go

@@ -18,9 +18,7 @@ package v1alpha1
 
 const (
 	PodScalingInPhaseLabel = "cafed.kusionstack.io/scaling-in" // indicate a pod is scaling in
-)
 
-const (
 	PodOperatingLabelPrefix           = "operating.kafed.kusionstack.io"            // indicate a pod is operating
 	PodOperationTypeLabelPrefix       = "operation-type.kafed.kusionstack.io"       // indicate the type of operation
 	PodOperationPermissionLabelPrefix = "operation-permission.kafed.kusionstack.io" // indicate the permission of operation
@@ -35,6 +33,8 @@ const (
 	PodPostCheckLabelPrefix   = "post-check.lifecycle.kafed.kusionstack.io"   // indicate a pod is in post-check phase
 	PodPostCheckedLabelPrefix = "post-checked.lifecycle.kafed.kusionstack.io" // indicate a pod has finished post-check phase
 	PodCompleteLabelPrefix    = "complete.lifecycle.kafed.kusionstack.io"     // indicate a pod has finished all phases
+
+	PodServiceAvailableLabel = "kafed.kusionstack.io/service-available" // indicate a pod is available to serve
 )
 
 var (

pkg/webhook/server/generic/pod/opslifecycle/webhook.go

@@ -18,13 +18,15 @@ package opslifecycle
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"kusionstack.io/kafed/apis/apps/v1alpha1"
@@ -50,21 +52,17 @@ var (
 )
 
 type ReadyToUpgrade func(pod *corev1.Pod) (bool, []string, *time.Duration)
-type SatisfyExpectedFinalizers func(pod *corev1.Pod) (bool, []string, error)
 type TimeLabelValue func() string
 
 type OpsLifecycle struct {
-	readyToUpgrade            ReadyToUpgrade
-	satisfyExpectedFinalizers SatisfyExpectedFinalizers
+	readyToUpgrade ReadyToUpgrade
 
 	timeLabelValue TimeLabelValue
 }
 
-func New(readyToUpgrade ReadyToUpgrade, satisfyExpectedFinalizers SatisfyExpectedFinalizers) *OpsLifecycle {
+func New(readyToUpgrade ReadyToUpgrade) *OpsLifecycle {
 	return &OpsLifecycle{
-		readyToUpgrade:            readyToUpgrade,
-		satisfyExpectedFinalizers: satisfyExpectedFinalizers,
-
+		readyToUpgrade: readyToUpgrade,
 		timeLabelValue: func() string {
 			return strconv.FormatInt(time.Now().Unix(), 10)
 		},
@@ -227,32 +225,76 @@ func (lc *OpsLifecycle) Mutating(ctx context.Context, oldPod, newPod *corev1.Pod
 		if err != nil {
 			return err
 		}
-		if satisfied {
+		if satisfied { // all operations are done and all expected finalizers are satisfied, then remove all unuseful labels, and add service available label
 			for id := range newIdToLabelsMap {
 				for _, v := range []string{v1alpha1.PodOperatedLabelPrefix, v1alpha1.PodDoneOperationTypeLabelPrefix, v1alpha1.PodPostCheckLabelPrefix, v1alpha1.PodPostCheckedLabelPrefix, v1alpha1.PodCompleteLabelPrefix} {
 					delete(newPod.Labels, fmt.Sprintf("%s/%s", v, id))
 				}
 			}
+			lc.addLabelWithTime(newPod, v1alpha1.PodServiceAvailableLabel)
 		}
 	}
 
 	return nil
 }
 
-func (lc *OpsLifecycle) addLabelWithTime(newPod *corev1.Pod, key string) {
-	if newPod.Labels == nil {
-		newPod.Labels = make(map[string]string)
+func (lc *OpsLifecycle) addLabelWithTime(pod *corev1.Pod, key string) {
+	if pod.Labels == nil {
+		pod.Labels = make(map[string]string)
 	}
-	newPod.Labels[key] = lc.timeLabelValue()
+	pod.Labels[key] = lc.timeLabelValue()
 }
 
-func (lc *OpsLifecycle) addReadinessGates(newPod *corev1.Pod, conditionType corev1.PodConditionType) {
-	for _, v := range newPod.Spec.ReadinessGates {
+func (lc *OpsLifecycle) addReadinessGates(pod *corev1.Pod, conditionType corev1.PodConditionType) {
+	for _, v := range pod.Spec.ReadinessGates {
 		if v.ConditionType == conditionType {
 			return
 		}
 	}
-	newPod.Spec.ReadinessGates = append(newPod.Spec.ReadinessGates, corev1.PodReadinessGate{
+	pod.Spec.ReadinessGates = append(pod.Spec.ReadinessGates, corev1.PodReadinessGate{
 		ConditionType: conditionType,
 	})
 }
+
+func (lc *OpsLifecycle) satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) {
+	satisfy := true
+	var expectedFinalizer []string // expected finalizers that are not satisfied
+
+	availableConditions, err := lc.podAvailableConditions(pod)
+	if err != nil {
+		return satisfy, expectedFinalizer, err
+	}
+
+	if availableConditions != nil && len(availableConditions.ExpectedFinalizers) != 0 {
+		existFinalizers := sets.String{}
+		for _, finalizer := range pod.Finalizers {
+			existFinalizers.Insert(finalizer)
+		}
+
+		for _, finalizer := range availableConditions.ExpectedFinalizers {
+			if !existFinalizers.Has(finalizer) {
+				satisfy = false
+				expectedFinalizer = append(expectedFinalizer, finalizer)
+			}
+		}
+	}
+
+	return satisfy, expectedFinalizer, nil
+}
+
+func (lc *OpsLifecycle) podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAvailableConditions, error) {
+	if pod.Annotations == nil {
+		return nil, nil
+	}
+
+	anno, ok := pod.Annotations[v1alpha1.PodAvailableConditionsAnnotation]
+	if !ok {
+		return nil, nil
+	}
+
+	availableConditions := &v1alpha1.PodAvailableConditions{}
+	if err := json.Unmarshal([]byte(anno), availableConditions); err != nil {
+		return nil, err
+	}
+	return availableConditions, nil
+}

pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go

@@ -107,8 +107,7 @@ func TestMutating(t *testing.T) {
 		newPodLabels   map[string]string
 		expectedLabels map[string]string
 
-		readyToUpgrade            ReadyToUpgrade
-		satisfyExpectedFinalizers SatisfyExpectedFinalizers
+		readyToUpgrade ReadyToUpgrade
 
 		keyWords string // used to check the error message
 	}{
@@ -377,26 +376,6 @@ func TestMutating(t *testing.T) {
 			},
 		},
 
-		{
-			notes: "wait for removing finalizers",
-			newPodLabels: map[string]string{
-				fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, "123"):          "1402144848",
-				fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, "123"): "upgrade",
-				fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, "123"):         "1402144848",
-				fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckedLabelPrefix, "123"):       "1402144848",
-
-				fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, "123"): "1402144848",
-			},
-			expectedLabels: map[string]string{
-				fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, "123"):          "1402144848",
-				fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, "123"): "upgrade",
-				fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, "123"):         "1402144848",
-				fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckedLabelPrefix, "123"):       "1402144848",
-
-				fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, "123"): "1402144848",
-			},
-			satisfyExpectedFinalizers: satifyExpectedFinalizersReturnFalse,
-		},
 		{
 			notes: "all finished",
 			newPodLabels: map[string]string{
@@ -414,7 +393,7 @@ func TestMutating(t *testing.T) {
 
 				fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, "456"): "1402144848",
 			},
-			expectedLabels: map[string]string{},
+			expectedLabels: map[string]string{v1alpha1.PodServiceAvailableLabel: "1402144848"},
 		},
 	}
 
@@ -443,10 +422,6 @@ func TestMutating(t *testing.T) {
 		if opslifecycle.readyToUpgrade == nil {
 			opslifecycle.readyToUpgrade = readyToUpgradeReturnTrue
 		}
-		opslifecycle.satisfyExpectedFinalizers = v.satisfyExpectedFinalizers
-		if opslifecycle.satisfyExpectedFinalizers == nil {
-			opslifecycle.satisfyExpectedFinalizers = satifyExpectedFinalizersReturnTrue
-		}
 
 		t.Logf("notes: %s", v.notes)
 		err := opslifecycle.Mutating(context.Background(), oldPod, newPod, nil, nil)
@@ -465,14 +440,6 @@ func readyToUpgradeReturnTrue(pod *corev1.Pod) (bool, []string, *time.Duration)
 	return true, nil, nil
 }
 
-func satifyExpectedFinalizersReturnTrue(pod *corev1.Pod) (bool, []string, error) {
-	return true, nil, nil
-}
-
 func readyToUpgradeReturnFalse(pod *corev1.Pod) (bool, []string, *time.Duration) {
 	return false, nil, nil
 }
-
-func satifyExpectedFinalizersReturnFalse(pod *corev1.Pod) (bool, []string, error) {
-	return false, nil, nil
-}

pkg/webhook/server/generic/pod/pod_mutating_handler.go

@@ -45,10 +45,10 @@ type MutatingHandler struct {
 	opsLifecycle  *opslifecycle.OpsLifecycle
 }
 
-func NewMutatingHandler(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade, satisfyExpectedFinalizers opslifecycle.SatisfyExpectedFinalizers) *MutatingHandler {
+func NewMutatingHandler(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade) *MutatingHandler {
 	return &MutatingHandler{
 		needLifecycle: needLifecycle,
-		opsLifecycle:  opslifecycle.New(readyToUpgrade, satisfyExpectedFinalizers),
+		opsLifecycle:  opslifecycle.New(readyToUpgrade),
 	}
 }
 

pkg/webhook/server/generic/pod/pod_webhook.go

@@ -30,9 +30,9 @@ type PodWebhook struct {
 	validatingHandler *ValidatingHandler
 }
 
-func NewPodWebhook(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade, satisfyExpectedFinalizers opslifecycle.SatisfyExpectedFinalizers) *PodWebhook {
+func NewPodWebhook(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade) *PodWebhook {
 	return &PodWebhook{
-		mutatingHandler:   NewMutatingHandler(needLifecycle, readyToUpgrade, satisfyExpectedFinalizers),
+		mutatingHandler:   NewMutatingHandler(needLifecycle, readyToUpgrade),
 		validatingHandler: NewValidatingHandler(needLifecycle),
 	}
 }