diff --git a/pkg/webhook/add_ingress.go b/pkg/webhook/add_ingress.go new file mode 100644 index 0000000000..8f3c7e37d7 --- /dev/null +++ b/pkg/webhook/add_ingress.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/webhook/ingress/validating" +) + +func init() { + addHandlers(validating.HandlerMap) +} diff --git a/pkg/webhook/add_service.go b/pkg/webhook/add_service.go new file mode 100644 index 0000000000..f09c42822e --- /dev/null +++ b/pkg/webhook/add_service.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/webhook/service/validating" +) + +func init() { + addHandlers(validating.HandlerMap) +} diff --git a/pkg/webhook/ingress/validating/ingress_handler.go b/pkg/webhook/ingress/validating/ingress_handler.go new file mode 100644 index 0000000000..4ae379a126 --- /dev/null +++ b/pkg/webhook/ingress/validating/ingress_handler.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + + "k8s.io/klog/v2" + + admissionv1 "k8s.io/api/admission/v1" + networkingv1 "k8s.io/api/networking/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type IngressHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &IngressHandler{} + +// Handle handles admission requests. +func (h *IngressHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate ingress %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + obj := &networkingv1.Ingress{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if err := deletionprotection.ValidateIngressDeletion(h.Client, obj); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &IngressHandler{} + +func (h *IngressHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &IngressHandler{} + +func (h *IngressHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/ingress/validating/webhooks.go b/pkg/webhook/ingress/validating/webhooks.go new file mode 100644 index 0000000000..4510ddd6b9 --- /dev/null +++ b/pkg/webhook/ingress/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-ingress,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=networking.k8s.io,resources=ingresses,verbs=delete,versions=v1,name=vingress.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-ingress": &IngressHandler{}, + } +) diff --git a/pkg/webhook/service/validating/service_handler.go b/pkg/webhook/service/validating/service_handler.go new file mode 100644 index 0000000000..4306223fd0 --- /dev/null +++ b/pkg/webhook/service/validating/service_handler.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + + "k8s.io/klog/v2" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type ServiceHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &ServiceHandler{} + +// Handle handles admission requests. +func (h *ServiceHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate service %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + obj := &v1.Service{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if err := deletionprotection.ValidateServiceDeletion(h.Client, obj); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &ServiceHandler{} + +func (h *ServiceHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &ServiceHandler{} + +func (h *ServiceHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/service/validating/webhooks.go b/pkg/webhook/service/validating/webhooks.go new file mode 100644 index 0000000000..54bd204a52 --- /dev/null +++ b/pkg/webhook/service/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-service,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=services,verbs=delete,versions=v1,name=vservice.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-service": &ServiceHandler{}, + } +) diff --git a/pkg/webhook/util/deletionprotection/deletion_protection.go b/pkg/webhook/util/deletionprotection/deletion_protection.go index 7928643083..3d6fe48539 100644 --- a/pkg/webhook/util/deletionprotection/deletion_protection.go +++ b/pkg/webhook/util/deletionprotection/deletion_protection.go @@ -22,8 +22,10 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" kubecontroller "k8s.io/kubernetes/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/client" @@ -50,6 +52,73 @@ func ValidateWorkloadDeletion(obj metav1.Object, replicas *int32) error { return nil } +func ValidateServiceDeletion(c client.Client, service *v1.Service) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || service.DeletionTimestamp != nil { + return nil + } + switch val := service.Labels[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + case policyv1alpha1.DeletionProtectionTypeCascading: + endpoints := v1.Endpoints{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, &endpoints); err != nil { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for get endpoints error: %v", err) + } + var ipCount int + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + if ipCount > 0 { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s and endpoints ip count %d>0", policyv1alpha1.DeletionProtectionKey, val, ipCount) + } + default: + } + return nil +} + +func ValidateIngressDeletion(c client.Client, ingress *networkingv1.Ingress) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || ingress.DeletionTimestamp != nil { + return nil + } + switch val := ingress.Labels[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + case policyv1alpha1.DeletionProtectionTypeCascading: + var serviceName []string + for _, rule := range ingress.Spec.Rules { + for _, path := range rule.HTTP.Paths { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: ingress.Namespace, Name: path.Backend.Service.Name}, &v1.Service{}); err == nil { + serviceName = append(serviceName, path.Backend.Service.Name) + } + } + } + + endpoints := v1.Endpoints{} + var ipCount int + for i := range serviceName { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: ingress.Namespace, Name: serviceName[i]}, &endpoints); err != nil { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for get endpoints error: %v", err) + } + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + } + if ipCount > 0 { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s and endpoints ip count %d>0", policyv1alpha1.DeletionProtectionKey, val, ipCount) + } + default: + } + return nil +} + func ValidateNamespaceDeletion(c client.Client, namespace *v1.Namespace) error { if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || namespace.DeletionTimestamp != nil { return nil diff --git a/test/e2e/policy/deletionprotection.go b/test/e2e/policy/deletionprotection.go index 0fd46a2a27..fa4a89562a 100644 --- a/test/e2e/policy/deletionprotection.go +++ b/test/e2e/policy/deletionprotection.go @@ -29,6 +29,7 @@ import ( "github.com/openkruise/kruise/test/e2e/framework" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/resource" @@ -371,4 +372,173 @@ var _ = SIGDescribe("DeletionProtection", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) + + framework.KruiseDescribe("Service deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Service with Always") + name := "svc-" + randStr + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"owner": "foo"}, + Ports: []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + }, + }, + } + _, err := c.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service should be rejected") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Create a CloneSet match the Service and wait for pod ready") + tester := framework.NewCloneSetTester(c, kc, ns) + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{}) + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"spec":{"selector":{"owner":"%s"}}}`, cs.ObjectMeta.Name)), metav1.PatchOptions{}) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Patch the Service deletion to Cascading") + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, policyv1alpha1.DeletionProtectionKey, policyv1alpha1.DeletionProtectionTypeCascading)), metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service should be rejected") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Scale CloneSet replicas to 0") + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32Ptr(0) + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 5*time.Second, time.Second).Should(gomega.Equal(int32(0))) + + ginkgo.By("Delete the Service successfully") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + framework.KruiseDescribe("Ingress deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Ingress with Always") + name := "ing-" + randStr + pathType := networkingv1.PathTypePrefix + ing := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err := c.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress should be rejected") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Create a Service match the Ingress") + name = "svc-" + randStr + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"owner": "foo"}, + Ports: []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + }, + }, + } + _, err = c.NetworkingV1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.JSONPatchType, + []byte(fmt.Sprintf(`[{"op":"replace","path":"/spec/rules/0/http/paths/0/backend/service/name","value":"%s"}]`, svc.Name)), metav1.PatchOptions{}) + _, err = c.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create a CloneSet match the Service and wait for pod ready") + tester := framework.NewCloneSetTester(c, kc, ns) + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{}) + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"spec":{"selector":{"owner":"%s"}}}`, cs.ObjectMeta.Name)), metav1.PatchOptions{}) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Patch the Ingress deletion to Cascading") + _, err = c.NetworkingV1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, policyv1alpha1.DeletionProtectionKey, policyv1alpha1.DeletionProtectionTypeCascading)), metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress should be rejected") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Scale CloneSet replicas to 0") + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32Ptr(0) + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 5*time.Second, time.Second).Should(gomega.Equal(int32(0))) + + ginkgo.By("Delete the Ingress successfully") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) })