From 4c502141e07cd8032cb3d9c3d93363b1d751093b Mon Sep 17 00:00:00 2001 From: Leo Huang <77225753+huangchenzhao@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:53:48 +0800 Subject: [PATCH] feat: improve hostNetwork mode of NodePool by adding NodeAffinity to pods with specified annotation (#1935) (#1959) --- .../yurt-manager-auto-generated.yaml | 20 + .../apps/well_known_labels_annotations.go | 6 + pkg/yurthub/filter/constant.go | 18 +- .../filter/hostnetworkpropagation/filter.go | 160 ------ .../hostnetworkpropagation/filter_test.go | 514 ------------------ pkg/yurthub/filter/manager/manager.go | 2 - pkg/yurthub/filter/manager/manager_test.go | 18 - .../webhook/pod/v1alpha1/pod_default.go | 69 +++ .../webhook/pod/v1alpha1/pod_default_test.go | 330 +++++++++++ .../webhook/pod/v1alpha1/pod_handler.go | 57 ++ pkg/yurtmanager/webhook/server.go | 2 + 11 files changed, 490 insertions(+), 706 deletions(-) delete mode 100644 pkg/yurthub/filter/hostnetworkpropagation/filter.go delete mode 100644 pkg/yurthub/filter/hostnetworkpropagation/filter_test.go create mode 100644 pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go create mode 100644 pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go create mode 100644 pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 72c6445feda..1870d8b5035 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -652,6 +652,26 @@ webhooks: resources: - platformadmins sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-core-openyurt-io-v1-pod + failurePolicy: Ignore + name: mutate.core.v1.pod.openyurt.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/pkg/apis/apps/well_known_labels_annotations.go b/pkg/apis/apps/well_known_labels_annotations.go index 082ef47ee3f..03d5626beb1 100644 --- a/pkg/apis/apps/well_known_labels_annotations.go +++ b/pkg/apis/apps/well_known_labels_annotations.go @@ -44,3 +44,9 @@ const ( NodePoolHostNetworkLabel = "nodepool.openyurt.io/hostnetwork" NodePoolChangedEvent = "NodePoolChanged" ) + +// Pod related labels and annotations +const ( + // AnnotationExcludeHostNetworkPool indicates the pod don't want to be scheduled to nodes in hostNetwork mode NodePool + AnnotationExcludeHostNetworkPool = "apps.openyurt.io/exclude-host-network-pool" +) diff --git a/pkg/yurthub/filter/constant.go b/pkg/yurthub/filter/constant.go index 084788b3de3..9080778ec34 100644 --- a/pkg/yurthub/filter/constant.go +++ b/pkg/yurthub/filter/constant.go @@ -37,11 +37,6 @@ const ( // in order to make NodePort will not be listened by kube-proxy component in specified NodePool. NodePortIsolationFilterName = "nodeportisolation" - // HostNetworkPropagationFilterName filter is used to set pod.spec.HostNetwork to true when the - // hostNetwork field(nodePool.spec.HostNetwork) is true. this is equivalent to the nodepool - // propagating the hostNetwork configuration to the pods running in it. - HostNetworkPropagationFilterName = "hostnetworkpropagation" - // SkipDiscardServiceAnnotation is annotation used by LB service. // If end users want to use specified LB service at the edge side, // End users should add annotation["openyurt.io/skip-discard"]="true" for LB service. @@ -55,15 +50,14 @@ const ( var ( // DisabledInCloudMode contains the filters that should be disabled when yurthub is working in cloud mode. - DisabledInCloudMode = []string{DiscardCloudServiceFilterName, HostNetworkPropagationFilterName} + DisabledInCloudMode = []string{DiscardCloudServiceFilterName} // SupportedComponentsForFilter is used for specifying which components are supported by filters as default setting. SupportedComponentsForFilter = map[string]string{ - MasterServiceFilterName: "kubelet", - DiscardCloudServiceFilterName: "kube-proxy", - ServiceTopologyFilterName: "kube-proxy, coredns, nginx-ingress-controller", - InClusterConfigFilterName: "kubelet", - NodePortIsolationFilterName: "kube-proxy", - HostNetworkPropagationFilterName: "kubelet", + MasterServiceFilterName: "kubelet", + DiscardCloudServiceFilterName: "kube-proxy", + ServiceTopologyFilterName: "kube-proxy, coredns, nginx-ingress-controller", + InClusterConfigFilterName: "kubelet", + NodePortIsolationFilterName: "kube-proxy", } ) diff --git a/pkg/yurthub/filter/hostnetworkpropagation/filter.go b/pkg/yurthub/filter/hostnetworkpropagation/filter.go deleted file mode 100644 index e6baf217f29..00000000000 --- a/pkg/yurthub/filter/hostnetworkpropagation/filter.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hostnetworkpropagation - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/filter" -) - -// Register registers a filter -func Register(filters *filter.Filters) { - filters.Register(filter.HostNetworkPropagationFilterName, func() (filter.ObjectFilter, error) { - return NewHostNetworkPropagationFilter() - }) -} - -func NewHostNetworkPropagationFilter() (filter.ObjectFilter, error) { - return &hostNetworkPropagationFilter{}, nil -} - -type hostNetworkPropagationFilter struct { - nodePoolLister cache.GenericLister - nodePoolSynced cache.InformerSynced - nodePoolName string - client kubernetes.Interface - isHostNetworkPool *bool -} - -func (hpf *hostNetworkPropagationFilter) Name() string { - return filter.HostNetworkPropagationFilterName -} - -func (hpf *hostNetworkPropagationFilter) SupportedResourceAndVerbs() map[string]sets.String { - return map[string]sets.String{ - "pods": sets.NewString("list", "watch"), - } -} - -func (hpf *hostNetworkPropagationFilter) SetNodePoolInformerFactory(dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory) error { - gvr := v1beta1.GroupVersion.WithResource("nodepools") - hpf.nodePoolLister = dynamicInformerFactory.ForResource(gvr).Lister() - hpf.nodePoolSynced = dynamicInformerFactory.ForResource(gvr).Informer().HasSynced - - return nil -} - -func (hpf *hostNetworkPropagationFilter) SetNodePoolName(poolName string) error { - hpf.nodePoolName = poolName - return nil -} - -func (hpf *hostNetworkPropagationFilter) SetKubeClient(client kubernetes.Interface) error { - hpf.client = client - return nil -} - -func (hpf *hostNetworkPropagationFilter) resolveNodePoolName(pod *corev1.Pod) string { - if len(hpf.nodePoolName) != 0 { - return hpf.nodePoolName - } - - node, err := hpf.client.CoreV1().Nodes().Get(context.Background(), pod.Spec.NodeName, metav1.GetOptions{}) - if err != nil { - klog.Warningf("could not get node(%s) in hostNetworkPropagationFilter, %v", pod.Spec.NodeName, err) - return hpf.nodePoolName - } - hpf.nodePoolName = node.Labels[projectinfo.GetNodePoolLabel()] - return hpf.nodePoolName -} - -func (hpf *hostNetworkPropagationFilter) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { - pod, ok := obj.(*corev1.Pod) - if !ok { - return obj - } - - // when pod hostnetwork is already set to true, the function will - // short-circuit and return - if pod.Spec.HostNetwork { - return obj - } - - if hpf.isHostNetworkPool == nil { - // go to configure IsHostNetworkPool - } else if *hpf.isHostNetworkPool { - pod.Spec.HostNetwork = true - return obj - } else { - // nodepool hostNetwork field is false, only short-circuit and return - return obj - } - - if ok := cache.WaitForCacheSync(stopCh, hpf.nodePoolSynced); !ok { - return obj - } - - nodePoolName := hpf.resolveNodePoolName(pod) - if len(nodePoolName) == 0 { - klog.Infof("node(%s) is not added into node pool, so skip hostnetworkpropagation", pod.Spec.NodeName) - return obj - } - - runtimeObj, err := hpf.nodePoolLister.Get(nodePoolName) - if err != nil { - klog.Warningf("hostNetworkPropagationFilter: could not get nodepool %s, err: %v", nodePoolName, err) - return obj - } - var nodePool *v1beta1.NodePool - switch poolObj := runtimeObj.(type) { - case *v1beta1.NodePool: - nodePool = poolObj - case *unstructured.Unstructured: - nodePool = new(v1beta1.NodePool) - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(poolObj.UnstructuredContent(), nodePool); err != nil { - klog.Warningf("hostNetworkPropagationFilter: object(%#+v) is not a v1beta1.NodePool", poolObj) - return obj - } - default: - klog.Warningf("object(%#+v) is not a unknown type", poolObj) - return obj - } - - if nodePool.Spec.HostNetwork { - valTrue := true - hpf.isHostNetworkPool = &valTrue - pod.Spec.HostNetwork = true - } else { - valFalse := false - hpf.isHostNetworkPool = &valFalse - } - - return obj -} diff --git a/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go b/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go deleted file mode 100644 index 7e363cb4c68..00000000000 --- a/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go +++ /dev/null @@ -1,514 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hostnetworkpropagation - -import ( - "reflect" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/dynamic/fake" - k8sfake "k8s.io/client-go/kubernetes/fake" - - "github.com/openyurtio/openyurt/pkg/apis" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/util" - "github.com/openyurtio/openyurt/pkg/yurthub/filter" -) - -func TestName(t *testing.T) { - hpf, _ := NewHostNetworkPropagationFilter() - if hpf.Name() != filter.HostNetworkPropagationFilterName { - t.Errorf("expect %s, but got %s", filter.HostNetworkPropagationFilterName, hpf.Name()) - } -} - -func TestSupportedResourceAndVerbs(t *testing.T) { - hpf, _ := NewHostNetworkPropagationFilter() - rvs := hpf.SupportedResourceAndVerbs() - if len(rvs) != 1 { - t.Errorf("supported not one resource, %v", rvs) - } - - for resource, verbs := range rvs { - if resource != "pods" { - t.Errorf("expect resource is pods, but got %s", resource) - } - - if !verbs.Equal(sets.NewString("list", "watch")) { - t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) - } - } -} - -func TestFilter(t *testing.T) { - scheme := runtime.NewScheme() - apis.AddToScheme(scheme) - gvrToListKind := map[schema.GroupVersionResource]string{ - {Group: "apps.openyurt.io", Version: "v1beta1", Resource: "nodepools"}: "NodePoolList", - } - - testcases := map[string]struct { - poolName string - responseObject []runtime.Object - kubeClient *k8sfake.Clientset - dynamicClient *fake.FakeDynamicClient - expectObject []runtime.Object - }{ - "pod hostnetwork is true": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - }, - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - }, - }, - "it is not a pod": { - responseObject: []runtime.Object{ - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - }, - }, - }, - expectObject: []runtime.Object{ - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - }, - }, - }, - }, - "pool hostNetwork is false with specified nodepool": { - poolName: "pool-foo", - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - }, - "pool hostNetwork true false with specified nodepool": { - poolName: "pool-foo", - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - HostNetwork: true, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - }, - }, - "pool hostNetwork is false without specified node pool": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "pool-foo", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - }, - "pool hostNetwork is true without specified node pool": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "pool-foo", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - HostNetwork: true, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - }, - }, - "pool hostNetwork is false with specified unknown node": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "unknown-node", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "pool-foo", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - HostNetwork: true, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "unknown-node", - }, - }, - }, - }, - "pool hostNetwork is false with unknown pool": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "unknown-pool", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - HostNetwork: true, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - }, - "two pods with hostnetwork nodepool": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-bar", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "pool-foo", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - HostNetwork: true, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-bar", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - HostNetwork: true, - }, - }, - }, - }, - "two pods with not hostnetwork nodepool": { - responseObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-bar", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - kubeClient: k8sfake.NewSimpleClientset( - &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-foo", - Labels: map[string]string{ - projectinfo.GetNodePoolLabel(): "pool-foo", - }, - }, - }, - ), - dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, - &v1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pool-foo", - }, - Spec: v1beta1.NodePoolSpec{ - Type: v1beta1.Edge, - }, - }, - ), - expectObject: []runtime.Object{ - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-foo", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-bar", - }, - Spec: corev1.PodSpec{ - NodeName: "node-foo", - }, - }, - }, - }, - } - - for k, tc := range testcases { - t.Run(k, func(t *testing.T) { - hpf := &hostNetworkPropagationFilter{ - nodePoolName: tc.poolName, - client: tc.kubeClient, - } - - if tc.dynamicClient != nil { - gvr := v1beta1.GroupVersion.WithResource("nodepools") - dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(tc.dynamicClient, 24*time.Hour) - nodePoolInformer := dynamicInformerFactory.ForResource(gvr) - nodePoolLister := nodePoolInformer.Lister() - nodePoolSynced := nodePoolInformer.Informer().HasSynced - - stopper := make(chan struct{}) - defer close(stopper) - dynamicInformerFactory.Start(stopper) - dynamicInformerFactory.WaitForCacheSync(stopper) - hpf.nodePoolLister = nodePoolLister - hpf.nodePoolSynced = nodePoolSynced - } - - stopCh := make(<-chan struct{}) - for i := range tc.responseObject { - newObj := hpf.Filter(tc.responseObject[i], stopCh) - if util.IsNil(newObj) { - t.Errorf("empty object is returned") - } - if !reflect.DeepEqual(newObj, tc.expectObject[i]) { - t.Errorf("hostNetworkPropagationFilter expect: \n%#+v\nbut got: \n%#+v\n", tc.expectObject, newObj) - } - } - }) - } -} diff --git a/pkg/yurthub/filter/manager/manager.go b/pkg/yurthub/filter/manager/manager.go index 0241d1a9e4c..19d559fd7fd 100644 --- a/pkg/yurthub/filter/manager/manager.go +++ b/pkg/yurthub/filter/manager/manager.go @@ -29,7 +29,6 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/filter/discardcloudservice" - "github.com/openyurtio/openyurt/pkg/yurthub/filter/hostnetworkpropagation" "github.com/openyurtio/openyurt/pkg/yurthub/filter/inclusterconfig" "github.com/openyurtio/openyurt/pkg/yurthub/filter/initializer" "github.com/openyurtio/openyurt/pkg/yurthub/filter/masterservice" @@ -137,5 +136,4 @@ func registerAllFilters(filters *filter.Filters) { discardcloudservice.Register(filters) inclusterconfig.Register(filters) nodeportisolation.Register(filters) - hostnetworkpropagation.Register(filters) } diff --git a/pkg/yurthub/filter/manager/manager_test.go b/pkg/yurthub/filter/manager/manager_test.go index 3ff79d6ce6c..9c4e6d31159 100644 --- a/pkg/yurthub/filter/manager/manager_test.go +++ b/pkg/yurthub/filter/manager/manager_test.go @@ -114,24 +114,6 @@ func TestFindResponseFilter(t *testing.T) { isFound: true, names: sets.NewString("nodeportisolation"), }, - "get hostnetwork propagation filter": { - enableResourceFilter: true, - accessServerThroughHub: true, - userAgent: "kubelet", - verb: "GET", - path: "/api/v1/pods", - isFound: true, - names: sets.NewString("hostnetworkpropagation"), - }, - "could not get hostnetwork propagation filter in cloud mode": { - enableResourceFilter: true, - accessServerThroughHub: true, - workingMode: "cloud", - userAgent: "kubelet", - verb: "GET", - path: "/api/v1/pods", - isFound: false, - }, } resolver := newTestRequestInfoResolver() diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go new file mode 100644 index 00000000000..25bf6283126 --- /dev/null +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openyurtio/openyurt/pkg/apis/apps" +) + +// Default implements builder.CustomDefaulter. +func (webhook *PodHandler) Default(ctx context.Context, obj runtime.Object, req admission.Request) error { + pod, ok := obj.(*corev1.Pod) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Pod but got a %T", obj)) + } + + // Add NodeAffinity to pods in order to avoid pods to be scheduled on the nodes in the hostNetwork mode NodePool + annotations := pod.ObjectMeta.GetAnnotations() + if annotations != nil && annotations[apps.AnnotationExcludeHostNetworkPool] == "true" { + excludeHostNetworkTerm := corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "nodepool.openyurt.io/hostnetwork", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"true"}, + }, + }, + } + + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + + if pod.Spec.Affinity.NodeAffinity == nil { + pod.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} + } + + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + excludeHostNetworkTerm, + ) + } + return nil +} diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go new file mode 100644 index 00000000000..0d67c94facd --- /dev/null +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go @@ -0,0 +1,330 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openyurtio/openyurt/pkg/apis/apps" +) + +func TestDefault(t *testing.T) { + testcases := map[string]struct { + obj runtime.Object + errHappened bool + wantedPod *corev1.Pod + }{ + "it is not a pod": { + obj: &corev1.Node{}, + errHappened: true, + }, + "pod with specified annotation but without Affinity": { + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + }, + wantedPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "nodepool.openyurt.io/hostnetwork", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"true"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "pod with specified annotation but without Affinity's NodeAffinity": { + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-test", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"value-test"}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + }, + wantedPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-test", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"value-test"}, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "nodepool.openyurt.io/hostnetwork", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"true"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "pod with specified annotation but without NodeAffinity's RequiredDuringSchedulingIgnoredDuringExecution": { + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ + { + Weight: 100, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "key-test", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"value-test"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantedPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ + { + Weight: 100, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "key-test", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"value-test"}, + }, + }, + }, + }, + }, + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "nodepool.openyurt.io/hostnetwork", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"true"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "pod with specified annotation, then append new informations to NodeAffinity's RequiredDuringSchedulingIgnoredDuringExecution": { + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "key-test", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"value-test"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantedPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + Annotations: map[string]string{ + apps.AnnotationExcludeHostNetworkPool: "true", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "key-test", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"value-test"}, + }, + }, + }, + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "nodepool.openyurt.io/hostnetwork", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"true"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "pod without specified annotation": { + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + }, + }, + wantedPod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-test", + Namespace: metav1.NamespaceDefault, + }, + }, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + h := PodHandler{} + err := h.Default(context.TODO(), tc.obj, admission.Request{}) + if tc.errHappened { + if err == nil { + t.Errorf("expect error, got nil") + } + } else if err != nil { + t.Errorf("expect no error, but got %v", err) + } else { + currentPod := tc.obj.(*corev1.Pod) + if !reflect.DeepEqual(currentPod, tc.wantedPod) { + t.Errorf("expect %#+v, got %#+v", tc.wantedPod, currentPod) + } + } + }) + } +} diff --git a/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go new file mode 100644 index 00000000000..6c8ae786442 --- /dev/null +++ b/pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" +) + +const ( + WebhookName = "pod" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validate path, error +func (webhook *PodHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + + gvk, err := apiutil.GVKForObject(&corev1.Pod{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + builder.WebhookManagedBy(mgr). + For(&corev1.Pod{}). + WithDefaulter(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-core-openyurt-io-v1-pod,mutating=true,failurePolicy=ignore,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=pods,verbs=create,versions=v1,name=mutate.core.v1.pod.openyurt.io + +// PodHandler implements a validating and defaulting webhook for Cluster. +type PodHandler struct { + Client client.Client +} + +var _ builder.CustomDefaulter = &PodHandler{} diff --git a/pkg/yurtmanager/webhook/server.go b/pkg/yurtmanager/webhook/server.go index 9989a153760..be6b725ef9f 100644 --- a/pkg/yurtmanager/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -36,6 +36,7 @@ import ( v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" v1alpha1platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha1" v1alpha2platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha2" + v1alpha1pod "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/pod/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller" v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1" @@ -81,6 +82,7 @@ func init() { addControllerWebhook(names.YurtAppOverriderController, &v1alpha1deploymentrender.DeploymentRenderHandler{}) independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{} + independentWebhooks[v1alpha1pod.WebhookName] = &v1alpha1pod.PodHandler{} } // Note !!! @kadisi