feat(webhook): add garcedelete webhook for PodOpsLifecycle (#134)

* feat(webhook): add garcedelete webhook for PodOpsLifecycle

* refactor(garcedeletewebhook): reuse PodDeleteOpsLifecycleAdapter, improved garcedeletewebhook return message

* feat(gracedelete webhook): add featuregate for gracedelete webhook

* style(gracedelete): optimize klog and annotation

config/webhook/webhook.yaml

@@ -76,6 +76,7 @@ webhooks:
         operations:
           - CREATE
           - UPDATE
+          - DELETE
         resources:
           - pods
         scope: '*'

go.mod

@@ -92,6 +92,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.28.3 // indirect
 	k8s.io/apiserver v0.22.6 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/kubectl v0.29.0
 	kusionstack.io/kube-api v0.0.27 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect

go.sum

@@ -1128,6 +1128,7 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82q
 k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kube-proxy v0.22.6/go.mod h1:xLxEZ3sHyz11XaRyxqI4Z4F3I/Wtt+Jlep8w5yxQPAY=
 k8s.io/kube-scheduler v0.22.6/go.mod h1:DcHj6ixvb0M1PvWFbg133a1pz/vv7OSCgZUDU/UUhlU=
+k8s.io/kubectl v0.22.6 h1:xUvvVuKpo1d2Pur7BtSEdf2wWhO5HuwYJSlpCP4pIJY=
 k8s.io/kubectl v0.22.6/go.mod h1:9ktAgMwUsd2w12Yhj/xhMZhNna1t9rfExJg9j9jCIYk=
 k8s.io/kubelet v0.22.6/go.mod h1:/nSfVw7oYzpmLn8Ua2q2Zix09Fq5gpDGnNqTbab9wts=
 k8s.io/kubernetes v1.22.6 h1:OPKNO4FElcN6wHc3N3P6uW3P1oHvzNxu+HJ8vGQtBzM=

pkg/features/features.go

@@ -26,10 +26,13 @@ import (
 const (
 	// AlibabaCloudSlb enables the alibaba_cloud_slb controller.
 	AlibabaCloudSlb featuregate.Feature = "AlibabaCloudSlb"
+	// GraceDeleteWebhook enables the gracedelete webhook
+	GraceDeleteWebhook featuregate.Feature = "GraceDeleteWebhook"
 )
 
 var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
-	AlibabaCloudSlb: {Default: false, PreRelease: featuregate.Alpha},
+	AlibabaCloudSlb:    {Default: false, PreRelease: featuregate.Alpha},
+	GraceDeleteWebhook: {Default: false, PreRelease: featuregate.Alpha},
 }
 
 func init() {

pkg/webhook/server/generic/pod/gracedelete/webhook.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2023 The KusionStack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gracedelete
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	"kusionstack.io/operating/apis/apps/v1alpha1"
+	"kusionstack.io/operating/pkg/controllers/poddeletion"
+	"kusionstack.io/operating/pkg/controllers/utils/podopslifecycle"
+	"kusionstack.io/operating/pkg/features"
+	"kusionstack.io/operating/pkg/utils"
+	"kusionstack.io/operating/pkg/utils/feature"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type GraceDelete struct {
+}
+
+func New() *GraceDelete {
+	return &GraceDelete{}
+}
+
+func (gd *GraceDelete) Name() string {
+	return "GraceDeleteWebhook"
+}
+
+func (gd *GraceDelete) Validating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error {
+	// GraceDeleteWebhook FeatureGate defaults to false
+	// Add '--feature-gates=GraceDeleteWebhook=true' to container args, to enable gracedelete webhook
+	if !feature.DefaultFeatureGate.Enabled(features.GraceDeleteWebhook) || operation != admissionv1.Delete {
+		return nil
+	}
+
+	pod := &corev1.Pod{}
+	if err := c.Get(ctx, types.NamespacedName{Namespace: oldPod.Namespace, Name: oldPod.Name}, pod); err != nil {
+		if !errors.IsNotFound(err) {
+			klog.Error(err, "failed to find pod")
+			return err
+		}
+
+		klog.V(2).Info("pod is deleted")
+		return nil
+	}
+	if !utils.ControlledByKusionStack(pod) {
+		return nil
+	}
+
+	// if Pod is not begin a deletion PodOpsLifecycle, trigger it
+	if !podopslifecycle.IsDuringOps(poddeletion.OpsLifecycleAdapter, pod) {
+		if _, err := podopslifecycle.Begin(c, poddeletion.OpsLifecycleAdapter, pod); err != nil {
+			return fmt.Errorf("fail to begin PodOpsLifecycle to delete Pod %s: %s", pod.Name, err)
+		}
+	}
+
+	// if Pod is allow to operate, delete it
+	if _, allowed := podopslifecycle.AllowOps(poddeletion.OpsLifecycleAdapter, 0, pod); !allowed {
+		var finalizers []string
+		for _, f := range pod.Finalizers {
+			if strings.HasPrefix(f, v1alpha1.PodOperationProtectionFinalizerPrefix) {
+				finalizers = append(finalizers, f)
+			}
+		}
+		return fmt.Errorf("podOpsLifecycle denied delete request, since related resources and finalizers have not been processed. Waiting for removing finalizers: %v", finalizers)
+	}
+	return nil
+}
+
+func (gd *GraceDelete) Mutating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error {
+	return nil
+}

pkg/webhook/server/generic/pod/gracedelete/webhook_test.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2023 The KusionStack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gracedelete
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/kubectl/pkg/scheme"
+	"kusionstack.io/operating/apis/apps/v1alpha1"
+	"kusionstack.io/operating/pkg/controllers/poddeletion"
+	"kusionstack.io/operating/pkg/utils/feature"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	admissionv1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestGraceDelete(t *testing.T) {
+
+	inputs := []struct {
+		keyWords string // used to check the error message
+		fakePod  corev1.Pod
+		oldPod   corev1.Pod
+
+		podLabels      map[string]string
+		expectedLabels map[string]string
+
+		reqOperation admissionv1.Operation
+	}{
+		{
+			reqOperation: admissionv1.Update,
+		},
+		{
+			reqOperation: admissionv1.Delete,
+		},
+		{
+			fakePod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test",
+				},
+			},
+			oldPod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test2",
+				},
+			},
+			reqOperation: admissionv1.Delete,
+		},
+		{
+			fakePod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test",
+					Labels: map[string]string{
+						fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
+					},
+				},
+			},
+			oldPod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test",
+					Labels: map[string]string{
+						fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey): "true",
+					},
+				},
+			},
+			expectedLabels: map[string]string{
+				fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()):     "testvalue",
+				fmt.Sprintf("%s/%s", v1alpha1.PodOperationTypeLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "testvalue",
+			},
+			keyWords:     "podOpsLifecycle denied",
+			reqOperation: admissionv1.Delete,
+		},
+		{
+			fakePod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test",
+					Labels: map[string]string{
+						fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey):                                         "true",
+						fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "true",
+					},
+				},
+			},
+			oldPod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "default",
+					Name:      "test",
+					Labels: map[string]string{
+						fmt.Sprintf(v1alpha1.ControlledByKusionStackLabelKey):                                         "true",
+						fmt.Sprintf("%s/%s", v1alpha1.PodOperateLabelPrefix, poddeletion.OpsLifecycleAdapter.GetID()): "true",
+					},
+				},
+			},
+			reqOperation: admissionv1.Delete,
+		},
+	}
+
+	gd := New()
+	runtime.Must(feature.DefaultMutableFeatureGate.Set("GraceDeleteWebhook=true"))
+	for _, v := range inputs {
+		client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(&v.fakePod).Build()
+		err := gd.Validating(context.Background(), client, &v.oldPod, nil, v.reqOperation)
+		if v.keyWords == "" {
+			assert.Nil(t, err)
+		} else {
+			assert.NotNil(t, err)
+			assert.Contains(t, err.Error(), v.keyWords)
+		}
+		if len(v.expectedLabels) != 0 {
+			pod := &corev1.Pod{}
+			client.Get(context.Background(), types.NamespacedName{Namespace: v.oldPod.Namespace, Name: v.oldPod.Name}, pod)
+			for k := range v.expectedLabels {
+				_, exist := pod.Labels[k]
+				assert.True(t, exist)
+			}
+		}
+	}
+}

pkg/webhook/server/generic/pod/opslifecycle/validating.go

@@ -30,7 +30,7 @@ import (
 )
 
 func (lc *OpsLifecycle) Validating(ctx context.Context, c client.Client, oldPod, newPod *corev1.Pod, operation admissionv1.Operation) error {
-	if !utils.ControlledByKusionStack(newPod) {
+	if operation == admissionv1.Delete || !utils.ControlledByKusionStack(newPod) {
 		return nil
 	}
 

pkg/webhook/server/generic/pod/pod_validating_handler.go

@@ -46,19 +46,17 @@ func NewValidatingHandler() *ValidatingHandler {
 }
 
 func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request) (resp admission.Response) {
-	if req.Operation == admissionv1.Delete {
-		return admission.Allowed("pod is allowed by opslifecycle")
-	}
-
 	logger := h.Logger.WithValues(
 		"op", req.Operation,
 		"pod", commonutils.AdmissionRequestObjectKeyString(req),
 	)
 
 	pod := &corev1.Pod{}
-	if err := h.Decoder.Decode(req, pod); err != nil {
-		s, _ := json.Marshal(req)
-		return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to decode old object from request %s: %s", s, err))
+	if req.Operation != admissionv1.Delete {
+		if err := h.Decoder.Decode(req, pod); err != nil {
+			s, _ := json.Marshal(req)
+			return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to decode old object from request %s: %s", s, err))
+		}
 	}
 
 	var oldPod *corev1.Pod

pkg/webhook/server/generic/pod/pod_webhook.go

@@ -23,6 +23,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"kusionstack.io/operating/pkg/webhook/server/generic/pod/gracedelete"
 	"kusionstack.io/operating/pkg/webhook/server/generic/pod/opslifecycle"
 	"kusionstack.io/operating/pkg/webhook/server/generic/pod/resourceconsist"
 )
@@ -39,6 +40,7 @@ type AdmissionWebhook interface {
 
 func init() {
 	webhooks = append(webhooks, opslifecycle.New())
+	webhooks = append(webhooks, gracedelete.New())
 	for _, podResourceConsistWebhook := range resourceconsist.PodResourceConsistWebhooks {
 		webhooks = append(webhooks, podResourceConsistWebhook)
 	}