From 43c376fb1e5501f292cd2e1c3b12be90cbc5f07c Mon Sep 17 00:00:00 2001 From: kevin1689 Date: Fri, 28 Apr 2023 01:45:19 +0100 Subject: [PATCH] add: Support the deletion protection of service and ingress Signed-off-by: kevin1689 --- config/webhook/manifests.yaml | 41 +++ pkg/webhook/add_ingress.go | 25 ++ pkg/webhook/add_service.go | 25 ++ .../ingress/validating/ingress_handler.go | 89 ++++++ pkg/webhook/ingress/validating/webhooks.go | 28 ++ .../service/validating/service_handler.go | 76 +++++ pkg/webhook/service/validating/webhooks.go | 28 ++ .../deletionprotection/deletion_protection.go | 92 ++++++ test/e2e/policy/deletionprotection.go | 293 ++++++++++++++++++ 9 files changed, 697 insertions(+) create mode 100644 pkg/webhook/add_ingress.go create mode 100644 pkg/webhook/add_service.go create mode 100644 pkg/webhook/ingress/validating/ingress_handler.go create mode 100644 pkg/webhook/ingress/validating/webhooks.go create mode 100644 pkg/webhook/service/validating/service_handler.go create mode 100644 pkg/webhook/service/validating/webhooks.go diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 7cd14510e7..72b8c435da 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -474,6 +474,27 @@ webhooks: resources: - imagepulljobs sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-ingress + failurePolicy: Fail + name: vingress.kb.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + - v1beta1 + operations: + - DELETE + resources: + - ingresses + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -619,6 +640,26 @@ webhooks: resources: - podunavailablebudgets sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-service + failurePolicy: Fail + name: vservice.kb.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - DELETE + resources: + - services + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/pkg/webhook/add_ingress.go b/pkg/webhook/add_ingress.go new file mode 100644 index 0000000000..8f3c7e37d7 --- /dev/null +++ b/pkg/webhook/add_ingress.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/webhook/ingress/validating" +) + +func init() { + addHandlers(validating.HandlerMap) +} diff --git a/pkg/webhook/add_service.go b/pkg/webhook/add_service.go new file mode 100644 index 0000000000..f09c42822e --- /dev/null +++ b/pkg/webhook/add_service.go @@ -0,0 +1,25 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/webhook/service/validating" +) + +func init() { + addHandlers(validating.HandlerMap) +} diff --git a/pkg/webhook/ingress/validating/ingress_handler.go b/pkg/webhook/ingress/validating/ingress_handler.go new file mode 100644 index 0000000000..84a98f8f59 --- /dev/null +++ b/pkg/webhook/ingress/validating/ingress_handler.go @@ -0,0 +1,89 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + + "k8s.io/klog/v2" + + admissionv1 "k8s.io/api/admission/v1" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type IngressHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &IngressHandler{} + +// Handle handles admission requests. +func (h *IngressHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate ingress %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + var metaObj metav1.Object + v1beta1 := &networkingv1beta1.Ingress{} + v1 := &networkingv1.Ingress{} + switch req.Kind.Version { + case "v1beta1": + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, v1beta1); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + metaObj = v1beta1 + case "v1": + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, v1); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + metaObj = v1 + } + + if err := deletionprotection.ValidateIngressDeletion(h.Client, metaObj, v1beta1, v1, req.Kind.Version); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &IngressHandler{} + +func (h *IngressHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &IngressHandler{} + +func (h *IngressHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/ingress/validating/webhooks.go b/pkg/webhook/ingress/validating/webhooks.go new file mode 100644 index 0000000000..df78babc5b --- /dev/null +++ b/pkg/webhook/ingress/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-ingress,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=networking.k8s.io,resources=ingresses,verbs=delete,versions=v1;v1beta1,name=vingress.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-ingress": &IngressHandler{}, + } +) diff --git a/pkg/webhook/service/validating/service_handler.go b/pkg/webhook/service/validating/service_handler.go new file mode 100644 index 0000000000..4306223fd0 --- /dev/null +++ b/pkg/webhook/service/validating/service_handler.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "net/http" + + "github.com/openkruise/kruise/pkg/webhook/util/deletionprotection" + + "k8s.io/klog/v2" + + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type ServiceHandler struct { + Client client.Client + + // Decoder decodes objects + Decoder *admission.Decoder +} + +var _ admission.Handler = &ServiceHandler{} + +// Handle handles admission requests. +func (h *ServiceHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + if req.AdmissionRequest.Operation != admissionv1.Delete || req.AdmissionRequest.SubResource != "" { + return admission.ValidationResponse(true, "") + } + if len(req.OldObject.Raw) == 0 { + klog.Warningf("Skip to validate service %s deletion for no old object, maybe because of Kubernetes version < 1.16", req.Name) + return admission.ValidationResponse(true, "") + } + + obj := &v1.Service{} + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, obj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if err := deletionprotection.ValidateServiceDeletion(h.Client, obj); err != nil { + return admission.Errored(http.StatusForbidden, err) + } + return admission.ValidationResponse(true, "") +} + +var _ inject.Client = &ServiceHandler{} + +func (h *ServiceHandler) InjectClient(c client.Client) error { + h.Client = c + return nil +} + +var _ admission.DecoderInjector = &ServiceHandler{} + +func (h *ServiceHandler) InjectDecoder(d *admission.Decoder) error { + h.Decoder = d + return nil +} diff --git a/pkg/webhook/service/validating/webhooks.go b/pkg/webhook/service/validating/webhooks.go new file mode 100644 index 0000000000..54bd204a52 --- /dev/null +++ b/pkg/webhook/service/validating/webhooks.go @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// +kubebuilder:webhook:path=/validate-service,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=services,verbs=delete,versions=v1,name=vservice.kb.io + +var ( + // HandlerMap contains admission webhook handlers + HandlerMap = map[string]admission.Handler{ + "validate-service": &ServiceHandler{}, + } +) diff --git a/pkg/webhook/util/deletionprotection/deletion_protection.go b/pkg/webhook/util/deletionprotection/deletion_protection.go index 7928643083..274cb0d656 100644 --- a/pkg/webhook/util/deletionprotection/deletion_protection.go +++ b/pkg/webhook/util/deletionprotection/deletion_protection.go @@ -22,8 +22,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" kubecontroller "k8s.io/kubernetes/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/client" @@ -50,6 +53,95 @@ func ValidateWorkloadDeletion(obj metav1.Object, replicas *int32) error { return nil } +func ValidateServiceDeletion(c client.Client, service *v1.Service) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || service.DeletionTimestamp != nil { + return nil + } + switch val := service.Labels[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + case policyv1alpha1.DeletionProtectionTypeCascading: + endpoints := v1.Endpoints{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: service.Namespace, Name: service.Name}, &endpoints); err != nil { + // a Service without selector will not automatically create the corresponding Endpoints, that kind of Service will not be protected + if err.Error() == fmt.Sprintf("Endpoints \"%s\" not found", service.Name) { + return nil + } else { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for get endpoints error: %v", err) + } + } + + var ipCount int + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + if ipCount > 0 { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s and endpoints ip count %d>0", policyv1alpha1.DeletionProtectionKey, val, ipCount) + } + default: + } + return nil +} + +func ValidateIngressDeletion(c client.Client, obj metav1.Object, ingressv1beta1 *networkingv1beta1.Ingress, ingressv1 *networkingv1.Ingress, version string) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || obj.GetDeletionTimestamp() != nil { + return nil + } + switch val := obj.GetLabels()[policyv1alpha1.DeletionProtectionKey]; val { + case policyv1alpha1.DeletionProtectionTypeAlways: + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s", policyv1alpha1.DeletionProtectionKey, val) + case policyv1alpha1.DeletionProtectionTypeCascading: + var serviceName []string + switch version { + case "v1beta1": + for _, rule := range ingressv1beta1.Spec.Rules { + for _, path := range rule.HTTP.Paths { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: ingressv1beta1.Namespace, Name: path.Backend.ServiceName}, &v1.Service{}); err == nil { + serviceName = append(serviceName, path.Backend.ServiceName) + } + } + } + case "v1": + for _, rule := range ingressv1.Spec.Rules { + for _, path := range rule.HTTP.Paths { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: ingressv1.Namespace, Name: path.Backend.Service.Name}, &v1.Service{}); err == nil { + serviceName = append(serviceName, path.Backend.Service.Name) + } + } + } + } + + endpoints := v1.Endpoints{} + var ipCount int + for i := range serviceName { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: obj.GetNamespace(), Name: serviceName[i]}, &endpoints); err != nil { + if err.Error() == fmt.Sprintf("Endpoints \"%s\" not found", serviceName[i]) { + return nil + } else { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for get endpoints error: %v", err) + } + } + + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + } + if ipCount > 0 { + return fmt.Errorf("forbidden by ResourcesProtectionDeletion for %s=%s and endpoints ip count %d>0", policyv1alpha1.DeletionProtectionKey, val, ipCount) + } + default: + } + return nil +} + func ValidateNamespaceDeletion(c client.Client, namespace *v1.Namespace) error { if !utilfeature.DefaultFeatureGate.Enabled(features.ResourcesDeletionProtection) || namespace.DeletionTimestamp != nil { return nil diff --git a/test/e2e/policy/deletionprotection.go b/test/e2e/policy/deletionprotection.go index 0fd46a2a27..3c0256ef85 100644 --- a/test/e2e/policy/deletionprotection.go +++ b/test/e2e/policy/deletionprotection.go @@ -29,6 +29,8 @@ import ( "github.com/openkruise/kruise/test/e2e/framework" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + networkingv1beta1 "k8s.io/api/networking/v1beta1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/resource" @@ -36,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" @@ -371,4 +374,294 @@ var _ = SIGDescribe("DeletionProtection", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) + + framework.KruiseDescribe("Service deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Service with Always") + name := "svc-" + randStr + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"owner": "foo"}, + Ports: []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + }, + }, + } + _, err := c.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service should be rejected") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Create a CloneSet match the Service and wait for pod ready") + tester := framework.NewCloneSetTester(c, kc, ns) + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{}) + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"spec":{"selector":{"owner":"%s"}}}`, cs.ObjectMeta.Name)), metav1.PatchOptions{}) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for Endpoints IP of the Service create") + gomega.Eventually(func() int32 { + endpoints, err := c.CoreV1().Endpoints(ns).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var ipCount int32 + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + return ipCount + }, 10*time.Second, time.Second).Should(gomega.Equal(int32(1))) + time.Sleep(time.Minute) + + ginkgo.By("Patch the Service deletion to Cascading") + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, policyv1alpha1.DeletionProtectionKey, policyv1alpha1.DeletionProtectionTypeCascading)), metav1.PatchOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Service should be rejected") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Scale CloneSet replicas to 0") + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32Ptr(0) + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 10*time.Second, time.Second).Should(gomega.Equal(int32(0))) + + ginkgo.By("Wait Endpoints IP of the Service become to 0") + gomega.Eventually(func() int32 { + endpoints, err := c.CoreV1().Endpoints(ns).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var ipCount int32 + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + return ipCount + }, 30*time.Second, time.Second).Should(gomega.Equal(int32(0))) + time.Sleep(time.Minute) + + ginkgo.By("Delete the Service successfully") + err = c.CoreV1().Services(ns).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) + + framework.KruiseDescribe("Ingress deletion", func() { + ginkgo.It("should be protected", func() { + ginkgo.By("Create a Ingress with Always") + name := "ing-" + randStr + pathType := networkingv1.PathTypePrefix + ing := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "test", + Port: networkingv1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err := c.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + + // for the cluster using old Kubernetes version, use networking.k8s.io/v1beta1 instead of networking.k8s.io/v1 to create Ingress resource + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + pathType := networkingv1beta1.PathTypePrefix + ing := &networkingv1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{policyv1alpha1.DeletionProtectionKey: policyv1alpha1.DeletionProtectionTypeAlways}, + }, + Spec: networkingv1beta1.IngressSpec{ + Rules: []networkingv1beta1.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networkingv1beta1.IngressRuleValue{ + HTTP: &networkingv1beta1.HTTPIngressRuleValue{ + Paths: []networkingv1beta1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: networkingv1beta1.IngressBackend{ + ServiceName: "test", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } + _, err = c.NetworkingV1beta1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress should be rejected") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + err = c.NetworkingV1beta1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + } + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Create a Service match the Ingress") + name = "svc-" + randStr + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"owner": "foo"}, + Ports: []v1.ServicePort{ + {Port: 80, Name: "http", Protocol: v1.ProtocolTCP}, + }, + }, + } + _, err = c.NetworkingV1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.JSONPatchType, + []byte(fmt.Sprintf(`[{"op":"replace","path":"/spec/rules/0/http/paths/0/backend/service/name","value":"%s"}]`, svc.Name)), metav1.PatchOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + _, err = c.NetworkingV1beta1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.JSONPatchType, + []byte(fmt.Sprintf(`[{"op":"replace","path":"/spec/rules/0/http/paths/0/backend/serviceName","value":"%s"}]`, svc.Name)), metav1.PatchOptions{}) + } + _, err = c.CoreV1().Services(ns).Create(context.TODO(), svc, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Create a CloneSet match the Service and wait for pod ready") + tester := framework.NewCloneSetTester(c, kc, ns) + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{}) + _, err = c.CoreV1().Services(ns).Patch(context.TODO(), svc.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"spec":{"selector":{"owner":"%s"}}}`, cs.ObjectMeta.Name)), metav1.PatchOptions{}) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 120*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for Endpoints IP of the Service create") + gomega.Eventually(func() int32 { + endpoints, err := c.CoreV1().Endpoints(ns).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var ipCount int32 + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + return ipCount + }, 10*time.Second, time.Second).Should(gomega.Equal(int32(1))) + time.Sleep(time.Minute) + + ginkgo.By("Patch the Ingress deletion to Cascading") + _, err = c.NetworkingV1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, policyv1alpha1.DeletionProtectionKey, policyv1alpha1.DeletionProtectionTypeCascading)), metav1.PatchOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + _, err = c.NetworkingV1beta1().Ingresses(ns).Patch(context.TODO(), ing.Name, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"metadata":{"labels":{"%s":"%s"}}}`, policyv1alpha1.DeletionProtectionKey, policyv1alpha1.DeletionProtectionTypeCascading)), metav1.PatchOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Delete the Ingress should be rejected") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + err = c.NetworkingV1beta1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + } + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(err.Error()).Should(gomega.ContainSubstring(deleteForbiddenMessage)) + + ginkgo.By("Scale CloneSet replicas to 0") + err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + cs.Spec.Replicas = utilpointer.Int32Ptr(0) + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 10*time.Second, time.Second).Should(gomega.Equal(int32(0))) + + ginkgo.By("Wait Endpoints IP of the Ingress become to 0") + gomega.Eventually(func() int32 { + endpoints, err := c.CoreV1().Endpoints(ns).Get(context.TODO(), svc.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var ipCount int32 + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + if address.IP != "" { + ipCount++ + } + } + } + return ipCount + }, 30*time.Second, time.Second).Should(gomega.Equal(int32(0))) + time.Sleep(time.Minute) + + ginkgo.By("Delete the Ingress successfully") + err = c.NetworkingV1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + if err != nil && err.Error() == "the server could not find the requested resource" { + err = nil + err = c.NetworkingV1beta1().Ingresses(ns).Delete(context.TODO(), ing.Name, metav1.DeleteOptions{}) + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) })