Skip to content

Commit

Permalink
refactor(garcedelete webhook): refactor garcedelete webhook with podd… (
Browse files Browse the repository at this point in the history 
#146)

* refactor(garcedelete webhook): refactor garcedelete webhook with poddeletion_controller

* fix(gracedelete webhook): set PodDeletionIndicationLabel value to a timestamp
  • Loading branch information
@cyh-ant
cyh-ant committed Jan 16, 2024
1 parent 08cd567 commit f2fabbb
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 39 deletions.
58 changes: 31 additions & 27 deletions pkg/webhook/server/generic/pod/gracedelete/webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,16 @@ package gracedelete
import (
"context"
"fmt"
"strconv"
"strings"
"time"

admissionv1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
"k8s.io/client-go/util/retry"
"kusionstack.io/operating/apis/apps/v1alpha1"
appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1"
"kusionstack.io/operating/pkg/controllers/poddeletion"
"kusionstack.io/operating/pkg/controllers/utils/podopslifecycle"
"kusionstack.io/operating/pkg/features"
Expand All @@ -49,42 +51,44 @@ func (gd *GraceDelete) Name() string {
func (gd *GraceDelete) Validating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error {
// GraceDeleteWebhook FeatureGate defaults to false
// Add '--feature-gates=GraceDeleteWebhook=true' to container args, to enable gracedelete webhook
if !feature.DefaultFeatureGate.Enabled(features.GraceDeleteWebhook) || operation != admissionv1.Delete {
if !feature.DefaultFeatureGate.Enabled(features.GraceDeleteWebhook) || operation != admissionv1.Delete || !utils.ControlledByKusionStack(oldPod) {
return nil
}

pod := &corev1.Pod{}
if err := c.Get(ctx, types.NamespacedName{Namespace: oldPod.Namespace, Name: oldPod.Name}, pod); err != nil {
if !errors.IsNotFound(err) {
klog.Error(err, "failed to find pod")
return err
}

klog.V(2).Info("pod is deleted")
return nil
}
if !utils.ControlledByKusionStack(pod) {
// if pod is allowed to delete
if _, allowed := podopslifecycle.AllowOps(poddeletion.OpsLifecycleAdapter, 0, oldPod); allowed {
return nil
}

// if Pod is not begin a deletion PodOpsLifecycle, trigger it
if !podopslifecycle.IsDuringOps(poddeletion.OpsLifecycleAdapter, pod) {
if _, err := podopslifecycle.Begin(c, poddeletion.OpsLifecycleAdapter, pod); err != nil {
return fmt.Errorf("fail to begin PodOpsLifecycle to delete Pod %s: %s", pod.Name, err)
// label pod to trigger poddeletion_controller reconcile
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
newPod := &corev1.Pod{}
err := c.Get(ctx, types.NamespacedName{Namespace: oldPod.Namespace, Name: oldPod.Name}, newPod)
if err != nil {
return err
}
if newPod.Labels == nil {
newPod.Labels = map[string]string{}
}
if _, ok := newPod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey]; ok {
return nil
}
newPod.Labels[appsv1alpha1.PodDeletionIndicationLabelKey] = strconv.FormatInt(time.Now().Unix(), 10)

return c.Update(ctx, newPod)
})

if err != nil {
return err
}

// if Pod is allow to operate, delete it
if _, allowed := podopslifecycle.AllowOps(poddeletion.OpsLifecycleAdapter, 0, pod); !allowed {
var finalizers []string
for _, f := range pod.Finalizers {
if strings.HasPrefix(f, v1alpha1.PodOperationProtectionFinalizerPrefix) {
finalizers = append(finalizers, f)
}
var finalizers []string
for _, f := range oldPod.Finalizers {
if strings.HasPrefix(f, v1alpha1.PodOperationProtectionFinalizerPrefix) {
finalizers = append(finalizers, f)
}
return fmt.Errorf("podOpsLifecycle denied delete request, since related resources and finalizers have not been processed. Waiting for removing finalizers: %v", finalizers)
}
return nil
return fmt.Errorf("podOpsLifecycle denied delete request, since related resources and finalizers have not been processed. Waiting for removing finalizers: %v", finalizers)
}

func (gd *GraceDelete) Mutating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error {
Expand Down
47 changes: 35 additions & 12 deletions pkg/webhook/server/generic/pod/gracedelete/webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ import (
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/kubectl/pkg/scheme"
"kusionstack.io/operating/apis/apps/v1alpha1"
"kusionstack.io/operating/pkg/controllers/poddeletion"
"kusionstack.io/operating/pkg/utils/feature"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

admissionv1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1"
)

func TestGraceDelete(t *testing.T) {
Expand Down Expand Up @@ -63,33 +63,37 @@ func TestGraceDelete(t *testing.T) {
oldPod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test2",
Name: "test1",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
},
},
},
keyWords: "not found",
reqOperation: admissionv1.Delete,
},
{
fakePod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test",
Name: "test2",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
"operating.podopslifecycle.kusionstack.io/pod-delete": "1704865098763959176",
},
},
},
oldPod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test",
Name: "test2",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
},
},
},
expectedLabels: map[string]string{
fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "testvalue",
fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "testvalue",
appsv1alpha1.PodDeletionIndicationLabelKey: "true",
},
keyWords: "podOpsLifecycle denied",
reqOperation: admissionv1.Delete,
Expand All @@ -98,20 +102,39 @@ func TestGraceDelete(t *testing.T) {
fakePod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test",
Name: "test2",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "true",
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
},
},
},
oldPod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test",
Name: "test2",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
},
Finalizers: []string{"prot.podopslifecycle.kusionstack.io/finalizer1,prot.podopslifecycle.kusionstack.io/finalizer2"},
},
},
expectedLabels: map[string]string{
appsv1alpha1.PodDeletionIndicationLabelKey: "true",
},
keyWords: "podOpsLifecycle denied",
reqOperation: admissionv1.Delete,
},
{
fakePod: corev1.Pod{},
oldPod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "test3",
Labels: map[string]string{
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "true",
fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
"operating.podopslifecycle.kusionstack.io/pod-delete": "1704865098763959176",
"operation-type.podopslifecycle.kusionstack.io/pod-delete": "1704865098763959336",
"operate.podopslifecycle.kusionstack.io/pod-delete": "1704865212856080006",
},
},
},
Expand Down

0 comments on commit f2fabbb

Please sign in to comment.