diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2a508f7dbb..985117e47b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,6 +75,7 @@ jobs: case: - kwok - kwok-with-cni + - kwok-single-node continue-on-error: false runs-on: ubuntu-latest steps: diff --git a/kustomize/kwok/deployment.yaml b/kustomize/kwok/deployment.yaml index 315b96b2cf..ab7193ebbd 100644 --- a/kustomize/kwok/deployment.yaml +++ b/kustomize/kwok/deployment.yaml @@ -14,6 +14,7 @@ spec: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - --manage-nodes-with-label-selector= + - --manage-single-node= - --disregard-status-with-annotation-selector=kwok.x-k8s.io/status=custom - --disregard-status-with-label-selector= - --node-ip=$(POD_IP) diff --git a/pkg/apis/config/v1alpha1/kwok_configuration_types.go b/pkg/apis/config/v1alpha1/kwok_configuration_types.go index 3eb985c803..633c95945e 100644 --- a/pkg/apis/config/v1alpha1/kwok_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwok_configuration_types.go @@ -70,19 +70,33 @@ type KwokConfigurationOptions struct { // is the default value for flag --tls-private-key-file TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty"` + // ManageSingleNode is the option to manage a single node name. + // is the default value for flag --manage-single-node + // Note: when `manage-all-nodes` is specified as true or + // `manage-nodes-with-label-selector` or `manage-nodes-with-annotation-selector` is specified, + // this is a no-op. + ManageSingleNode string `json:"manageSingleNode,omitempty"` + // Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. // is the default value for flag --manage-all-nodes + // Note: when `manage-single-node` is specified as true or + // `manage-nodes-with-label-selector` or `manage-nodes-with-annotation-selector` is specified, + // this is a no-op. // +default=false ManageAllNodes *bool `json:"manageAllNodes,omitempty"` // Default annotations specified on Nodes to demand manage. - // Note: when `all-node-manage` is specified as true, this is a no-op. // is the default value for flag --manage-nodes-with-annotation-selector + // Note: when `all-node-manage` is specified as true or + // `manage-single-node` is specified, + // this is a no-op. ManageNodesWithAnnotationSelector string `json:"manageNodesWithAnnotationSelector,omitempty"` // Default labels specified on Nodes to demand manage. - // Note: when `all-node-manage` is specified as true, this is a no-op. // is the default value for flag --manage-nodes-with-label-selector + // Note: when `all-node-manage` is specified as true or + // `manage-single-node` is specified, + // this is a no-op. ManageNodesWithLabelSelector string `json:"manageNodesWithLabelSelector,omitempty"` // If a Node/Pod is on a managed Node and has this annotation status will not be modified diff --git a/pkg/apis/internalversion/kwok_configuration_types.go b/pkg/apis/internalversion/kwok_configuration_types.go index e100a09cdc..57b8729f9e 100644 --- a/pkg/apis/internalversion/kwok_configuration_types.go +++ b/pkg/apis/internalversion/kwok_configuration_types.go @@ -53,6 +53,9 @@ type KwokConfigurationOptions struct { // TLSPrivateKeyFile is the ile containing x509 private key TLSPrivateKeyFile string + // ManageSingleNode is the option to manage a single node name + ManageSingleNode string + // Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. ManageAllNodes bool @@ -74,11 +77,11 @@ type KwokConfigurationOptions struct { // Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux. EnableCNI bool - // enableDebuggingHandlers enables server endpoints for log collection + // EnableDebuggingHandlers enables server endpoints for log collection // and local running of containers and commands EnableDebuggingHandlers bool - // enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. + // EnableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. EnableContentionProfiling bool // EnableProfiling enables /debug/pprof handler. diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 92b9ff6161..50bede41b2 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1231,6 +1231,7 @@ func autoConvert_internalversion_KwokConfigurationOptions_To_v1alpha1_KwokConfig out.NodePort = in.NodePort out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.ManageSingleNode = in.ManageSingleNode if err := v1.Convert_bool_To_Pointer_bool(&in.ManageAllNodes, &out.ManageAllNodes, s); err != nil { return err } @@ -1271,6 +1272,7 @@ func autoConvert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfig out.NodePort = in.NodePort out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.ManageSingleNode = in.ManageSingleNode if err := v1.Convert_Pointer_bool_To_bool(&in.ManageAllNodes, &out.ManageAllNodes, s); err != nil { return err } diff --git a/pkg/kwok/cmd/root.go b/pkg/kwok/cmd/root.go index d88d916a87..2720ee669b 100644 --- a/pkg/kwok/cmd/root.go +++ b/pkg/kwok/cmd/root.go @@ -79,9 +79,10 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().IntVar(&flags.Options.NodePort, "node-port", flags.Options.NodePort, "Port of the node") cmd.Flags().StringVar(&flags.Options.TLSCertFile, "tls-cert-file", flags.Options.TLSCertFile, "File containing the default x509 Certificate for HTTPS") cmd.Flags().StringVar(&flags.Options.TLSPrivateKeyFile, "tls-private-key-file", flags.Options.TLSPrivateKeyFile, "File containing the default x509 private key matching --tls-cert-file") - cmd.Flags().BoolVar(&flags.Options.ManageAllNodes, "manage-all-nodes", flags.Options.ManageAllNodes, "All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector and manage-nodes-with-label-selector.") - cmd.Flags().StringVar(&flags.Options.ManageNodesWithAnnotationSelector, "manage-nodes-with-annotation-selector", flags.Options.ManageNodesWithAnnotationSelector, "Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes.") - cmd.Flags().StringVar(&flags.Options.ManageNodesWithLabelSelector, "manage-nodes-with-label-selector", flags.Options.ManageNodesWithLabelSelector, "Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes.") + cmd.Flags().StringVar(&flags.Options.ManageSingleNode, "manage-single-node", flags.Options.ManageSingleNode, "Node that matches the name will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-all-nodes.") + cmd.Flags().BoolVar(&flags.Options.ManageAllNodes, "manage-all-nodes", flags.Options.ManageAllNodes, "All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-single-node.") + cmd.Flags().StringVar(&flags.Options.ManageNodesWithAnnotationSelector, "manage-nodes-with-annotation-selector", flags.Options.ManageNodesWithAnnotationSelector, "Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node.") + cmd.Flags().StringVar(&flags.Options.ManageNodesWithLabelSelector, "manage-nodes-with-label-selector", flags.Options.ManageNodesWithLabelSelector, "Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node.") cmd.Flags().StringVar(&flags.Options.DisregardStatusWithAnnotationSelector, "disregard-status-with-annotation-selector", flags.Options.DisregardStatusWithAnnotationSelector, "All node/pod status excluding the ones that match the annotation selector will be watched and managed.") cmd.Flags().StringVar(&flags.Options.DisregardStatusWithLabelSelector, "disregard-status-with-label-selector", flags.Options.DisregardStatusWithLabelSelector, "All node/pod status excluding the ones that match the label selector will be watched and managed.") cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "Path to the kubeconfig file to use") @@ -229,24 +230,6 @@ func runE(ctx context.Context, flags *flagpole) error { return err } - if flags.Options.ManageAllNodes { - if flags.Options.ManageNodesWithAnnotationSelector != "" || flags.Options.ManageNodesWithLabelSelector != "" { - logger.Error("manage-all-nodes is conflicted with manage-nodes-with-annotation-selector and manage-nodes-with-label-selector.", nil) - os.Exit(1) - } - logger.Info("Watch all nodes") - } else if flags.Options.ManageNodesWithAnnotationSelector != "" || flags.Options.ManageNodesWithLabelSelector != "" { - logger.Info("Watch nodes", - "annotation", flags.Options.ManageNodesWithAnnotationSelector, - "label", flags.Options.ManageNodesWithLabelSelector, - ) - } - - err = waitForReady(ctx, typedClient) - if err != nil { - return err - } - id, err := controllers.Identity() if err != nil { return err @@ -261,6 +244,7 @@ func runE(ctx context.Context, flags *flagpole) error { EnableCNI: flags.Options.EnableCNI, EnableMetrics: enableMetrics, EnablePodCache: enableMetrics, + ManageSingleNode: flags.Options.ManageSingleNode, ManageAllNodes: flags.Options.ManageAllNodes, ManageNodesWithAnnotationSelector: flags.Options.ManageNodesWithAnnotationSelector, ManageNodesWithLabelSelector: flags.Options.ManageNodesWithLabelSelector, @@ -282,6 +266,11 @@ func runE(ctx context.Context, flags *flagpole) error { return err } + err = waitForReady(ctx, typedClient) + if err != nil { + return err + } + serverAddress := flags.Options.ServerAddress if serverAddress == "" && flags.Options.NodePort != 0 { serverAddress = "0.0.0.0:" + format.String(flags.Options.NodePort) diff --git a/pkg/kwok/controllers/controller.go b/pkg/kwok/controllers/controller.go index e988938915..966273bfb6 100644 --- a/pkg/kwok/controllers/controller.go +++ b/pkg/kwok/controllers/controller.go @@ -112,6 +112,7 @@ type Config struct { EnableCNI bool TypedClient kubernetes.Interface TypedKwokClient versioned.Interface + ManageSingleNode string ManageAllNodes bool ManageNodesWithAnnotationSelector string ManageNodesWithLabelSelector string @@ -132,16 +133,31 @@ type Config struct { EnablePodCache bool } -// NewController creates a new fake kubelet controller -func NewController(conf Config) (*Controller, error) { +func (c Config) validate() error { switch { - case conf.ManageAllNodes: - conf.ManageNodesWithAnnotationSelector = "" - conf.ManageNodesWithLabelSelector = "" - case conf.ManageNodesWithAnnotationSelector != "": - case conf.ManageNodesWithLabelSelector != "": + case c.ManageSingleNode != "": + if c.ManageAllNodes { + return fmt.Errorf("manage-single-node is conflicted with manage-all-nodes") + } + if c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "" { + return fmt.Errorf("manage-single-node is conflicted with manage-nodes-with-annotation-selector or manage-nodes-with-label-selector") + } + case c.ManageAllNodes: + if c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "" { + return fmt.Errorf("manage-all-nodes is conflicted with manage-nodes-with-annotation-selector or manage-nodes-with-label-selector") + } + case c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "": default: - return nil, fmt.Errorf("no nodes are managed") + return fmt.Errorf("no nodes are managed") + } + return nil +} + +// NewController creates a new fake kubelet controller +func NewController(conf Config) (*Controller, error) { + err := conf.validate() + if err != nil { + return nil, err } n := &Controller{ @@ -170,14 +186,34 @@ func (c *Controller) Start(ctx context.Context) error { onLeaseNodeManageFunc func(nodeName string) onNodeManagedFunc func(nodeName string) readOnlyFunc func(nodeName string) bool + + manageNodesWithLabelSelector string + manageNodesWithAnnotationSelector string + manageNodesWithFieldSelector string + manageNodeLeasesWithFieldSelector string + managePodsWithFieldSelector string ) + switch { + case conf.ManageSingleNode != "": + managePodsWithFieldSelector = fields.OneTermEqualSelector("spec.nodeName", conf.ManageSingleNode).String() + manageNodesWithFieldSelector = fields.OneTermEqualSelector("metadata.name", conf.ManageSingleNode).String() + manageNodeLeasesWithFieldSelector = fields.OneTermEqualSelector("metadata.name", conf.ManageSingleNode).String() + case conf.ManageAllNodes: + managePodsWithFieldSelector = fields.OneTermNotEqualSelector("spec.nodeName", "").String() + case conf.ManageNodesWithLabelSelector != "" || conf.ManageNodesWithAnnotationSelector != "": + manageNodesWithLabelSelector = conf.ManageNodesWithLabelSelector + manageNodesWithAnnotationSelector = conf.ManageNodesWithAnnotationSelector + managePodsWithFieldSelector = fields.OneTermNotEqualSelector("spec.nodeName", "").String() + } + nodeChan := make(chan informer.Event[*corev1.Node], 1) nodesCli := conf.TypedClient.CoreV1().Nodes() nodesInformer := informer.NewInformer[*corev1.Node, *corev1.NodeList](nodesCli) nodesCache, err := nodesInformer.WatchWithCache(ctx, informer.Option{ - LabelSelector: conf.ManageNodesWithLabelSelector, - AnnotationSelector: conf.ManageNodesWithAnnotationSelector, + LabelSelector: manageNodesWithLabelSelector, + AnnotationSelector: manageNodesWithAnnotationSelector, + FieldSelector: manageNodesWithFieldSelector, }, nodeChan) if err != nil { return fmt.Errorf("failed to watch nodes: %w", err) @@ -188,7 +224,7 @@ func (c *Controller) Start(ctx context.Context) error { podsInformer := informer.NewInformer[*corev1.Pod, *corev1.PodList](podsCli) podWatchOption := informer.Option{ - FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", "").String(), + FieldSelector: managePodsWithFieldSelector, } var podsCache informer.Getter[*corev1.Pod] @@ -205,7 +241,9 @@ func (c *Controller) Start(ctx context.Context) error { nodeLeasesChan = make(chan informer.Event[*coordinationv1.Lease], 1) nodeLeasesCli := conf.TypedClient.CoordinationV1().Leases(corev1.NamespaceNodeLease) nodeLeasesInformer := informer.NewInformer[*coordinationv1.Lease, *coordinationv1.LeaseList](nodeLeasesCli) - err = nodeLeasesInformer.Watch(ctx, informer.Option{}, nodeLeasesChan) + err = nodeLeasesInformer.Watch(ctx, informer.Option{ + FieldSelector: manageNodeLeasesWithFieldSelector, + }, nodeLeasesChan) if err != nil { return fmt.Errorf("failed to watch nodes: %w", err) } diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 0464b24f67..43520013df 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -1860,6 +1860,21 @@ is the default value for flag –tls-private-key-file

+manageSingleNode + +string + + + +

ManageSingleNode is the option to manage a single node name. +is the default value for flag –manage-single-node +Note: when manage-all-nodes is specified as true or +manage-nodes-with-label-selector or manage-nodes-with-annotation-selector is specified, +this is a no-op.

+ + + + manageAllNodes bool @@ -1867,7 +1882,10 @@ bool

Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. -is the default value for flag –manage-all-nodes

+is the default value for flag –manage-all-nodes +Note: when manage-single-node is specified as true or +manage-nodes-with-label-selector or manage-nodes-with-annotation-selector is specified, +this is a no-op.

@@ -1879,8 +1897,10 @@ string

Default annotations specified on Nodes to demand manage. -Note: when all-node-manage is specified as true, this is a no-op. -is the default value for flag –manage-nodes-with-annotation-selector

+is the default value for flag –manage-nodes-with-annotation-selector +Note: when all-node-manage is specified as true or +manage-single-node is specified, +this is a no-op.

@@ -1892,8 +1912,10 @@ string

Default labels specified on Nodes to demand manage. -Note: when all-node-manage is specified as true, this is a no-op. -is the default value for flag –manage-nodes-with-label-selector

+is the default value for flag –manage-nodes-with-label-selector +Note: when all-node-manage is specified as true or +manage-single-node is specified, +this is a no-op.

diff --git a/site/content/en/docs/generated/kwok.md b/site/content/en/docs/generated/kwok.md index 2fd4e7f5ca..7f90908a67 100644 --- a/site/content/en/docs/generated/kwok.md +++ b/site/content/en/docs/generated/kwok.md @@ -17,9 +17,10 @@ kwok [flags] --experimental-enable-cni Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux -h, --help help for kwok --kubeconfig string Path to the kubeconfig file to use (default "~/.kube/config") - --manage-all-nodes All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector and manage-nodes-with-label-selector. - --manage-nodes-with-annotation-selector string Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes. - --manage-nodes-with-label-selector string Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes. + --manage-all-nodes All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-single-node. + --manage-nodes-with-annotation-selector string Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node. + --manage-nodes-with-label-selector string Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node. + --manage-single-node string Node that matches the name will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-all-nodes. --master string The address of the Kubernetes API server (overrides any value in kubeconfig). --node-ip string IP of the node --node-lease-duration-seconds uint Duration of node lease seconds diff --git a/test/kwok-single-node/fake-deployment.yaml b/test/kwok-single-node/fake-deployment.yaml new file mode 100644 index 0000000000..bc63bf9614 --- /dev/null +++ b/test/kwok-single-node/fake-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fake-pod + namespace: default +spec: + replicas: 5 + selector: + matchLabels: + app: fake-pod + template: + metadata: + labels: + app: fake-pod + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: In + values: + - kwok-controller + tolerations: + - key: "kwok-controller/provider" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: fake-container + image: fake + nodeName: fake-node diff --git a/test/kwok-single-node/fake-node.yaml b/test/kwok-single-node/fake-node.yaml new file mode 100644 index 0000000000..08a654f9fc --- /dev/null +++ b/test/kwok-single-node/fake-node.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Node +metadata: + annotations: + kwok.x-k8s.io/node: fake + node.alpha.kubernetes.io/ttl: "0" + labels: + beta.kubernetes.io/arch: amd64 + beta.kubernetes.io/os: linux + kubernetes.io/arch: amd64 + kubernetes.io/hostname: fake-node + kubernetes.io/os: linux + kubernetes.io/role: agent + node-role.kubernetes.io/agent: "" + type: kwok-controller + name: fake-node +spec: + taints: + - effect: NoSchedule + key: kwok-controller/provider + value: fake diff --git a/test/kwok-single-node/kustomization.yaml b/test/kwok-single-node/kustomization.yaml new file mode 100644 index 0000000000..061e96a887 --- /dev/null +++ b/test/kwok-single-node/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +images: +- name: registry.k8s.io/kwok/kwok + newName: kwok + newTag: test + +patches: +- target: + group: apps + version: v1 + kind: Deployment + name: kwok-controller + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: --master=https://127.0.0.1:6443 + - op: replace + path: /spec/template/spec/hostNetwork + value: true + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --manage-nodes-with-annotation-selector= + - op: replace + path: /spec/template/spec/containers/0/args/3 + value: --manage-single-node=fake-node + +resources: +- ../../kustomize/kwok +- fake-deployment.yaml +- fake-node.yaml diff --git a/test/kwok-single-node/kwok-single-node.test.sh b/test/kwok-single-node/kwok-single-node.test.sh new file mode 100755 index 0000000000..ba24bf4bec --- /dev/null +++ b/test/kwok-single-node/kwok-single-node.test.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +DIR="$(realpath "${DIR}")" + +ROOT_DIR="$(realpath "${DIR}/../..")" + +CLUSTER_NAME=kwok-test +KWOK_IMAGE="kwok" +KWOK_VERSION="test" + +export PATH="${ROOT_DIR}/bin:${PATH}" + +function start_cluster() { + local linux_platform + linux_platform="linux/$(go env GOARCH)" + "${ROOT_DIR}"/hack/releases.sh --bin kwok --platform "${linux_platform}" + "${ROOT_DIR}"/images/kwok/build.sh --image "${KWOK_IMAGE}" --version="${KWOK_VERSION}" --platform "${linux_platform}" + + kind create cluster --name="${CLUSTER_NAME}" + + kind load docker-image --name="${CLUSTER_NAME}" "${KWOK_IMAGE}:${KWOK_VERSION}" + + kubectl kustomize "${DIR}" | kubectl apply -f - + kubectl kustomize "${ROOT_DIR}/stages" | kubectl apply -f - +} + +# Check for normal heartbeat +function test_node_ready() { + for ((i = 0; i < 30; i++)); do + if [[ ! "$(kubectl get node fake-node)" =~ "Ready" ]]; then + echo "Waiting for fake-node to be ready..." + sleep 1 + else + break + fi + done + + if [[ ! "$(kubectl get node fake-node)" =~ "Ready" ]]; then + echo "Error: fake-node is not ready" + kubectl get node fake-node + return 1 + fi +} + +# Check pod status +function test_check_pod_status() { + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + for row in $(kubectl get po -ojson | jq -r '.items[] | @base64'); do + pod_name=$(_jq '.metadata.name') + node_name=$(_jq '.spec.nodeName') + host_network=$(_jq '.spec.hostNetwork') + host_ip=$(_jq '.status.hostIP') + pod_ip=$(_jq '.status.podIP') + node_ip=$(kubectl get nodes "${node_name}" -ojson | jq -r '.status.addresses[] | select( .type == "InternalIP" ) | .address') + if [[ "${host_ip}" != "${node_ip}" ]]; then + echo "Error: ${pod_name} Pod: HostIP (${host_ip}) is not equal to the IP on its Node (${node_ip})" + return 1 + fi + if [[ $host_network == "true" ]]; then + if [[ "${pod_ip}" != "${node_ip}" ]]; then + echo "Error: ${pod_name} Pod with hostNetwork=${host_network}: PodIP (${pod_ip}) and HostIP (${host_ip}) are not equal" + return 1 + fi + fi + done + echo "Pod's status checked" +} + +# Check for the Pod is running +function test_pod_running() { + for ((i = 0; i < 30; i++)); do + if [[ ! "$(kubectl get pod | grep -c Running)" -eq 5 ]]; then + echo "Waiting all pods to be running..." + sleep 1 + else + break + fi + done + + if [[ ! "$(kubectl get pod | grep -c Running)" -eq 5 ]]; then + echo "Error: Not all pods are running" + kubectl get pod -o wide + return 1 + fi +} + +# Check for the status of the Node is modified by Kubectl +function test_modify_node_status() { + kubectl annotate node fake-node --overwrite kwok.x-k8s.io/status=custom + kubectl patch node fake-node --subresource=status -p '{"status":{"nodeInfo":{"kubeletVersion":"fake-new"}}}' + + sleep 2 + + if [[ ! "$(kubectl get node fake-node)" =~ "fake-new" ]]; then + echo "Error: fake-node is not updated" + kubectl get node fake-node + return 1 + fi + kubectl annotate node fake-node --overwrite kwok.x-k8s.io/status- +} + +# Check for the status of the Pod is modified by Kubectl +function test_modify_pod_status() { + local first_pod + first_pod="$(kubectl get pod | grep Running | head -n 1 | awk '{print $1}')" + + kubectl annotate pod "${first_pod}" --overwrite kwok.x-k8s.io/status=custom + kubectl patch pod "${first_pod}" --subresource=status -p '{"status":{"podIP":"192.168.0.1"}}' + + sleep 2 + + if [[ ! "$(kubectl get pod "${first_pod}" -o wide)" == *"192.168.0.1"* ]]; then + echo "Error: fake-pod is not updated" + kubectl get pod "${first_pod}" -o wide + return 1 + fi + + kubectl annotate pod "${first_pod}" --overwrite kwok.x-k8s.io/status- +} + +function test_check_node_lease_transitions() { + local want="${1}" + local node_leases_transitions + node_leases_transitions="$(kubectl get leases fake-node -n kube-node-lease -ojson | jq -r '.spec.leaseTransitions // 0')" + if [[ "${node_leases_transitions}" != "${want}" ]]; then + echo "Error: fake-node lease transitions is not ${want}, got ${node_leases_transitions}" + return 1 + fi +} + +function recreate_kwok() { + kubectl scale deployment/kwok-controller -n kube-system --replicas=0 + kubectl wait --for=delete pod -l app=kwok-controller -n kube-system --timeout=60s + + kubectl scale deployment/kwok-controller -n kube-system --replicas=2 +} + +function recreate_pods() { + kubectl delete pod --all -n default +} + +# Cleanup +function cleanup() { + kind delete cluster --name="${CLUSTER_NAME}" +} + +function main() { + failed=() + start_cluster || { + echo "Error: Failed to start cluster" + exit 1 + } + trap cleanup EXIT + + test_node_ready || failed+=("node_ready") + test_pod_running || failed+=("pod_running") + test_check_pod_status || failed+=("check_pod_status") + + test_modify_node_status || failed+=("modify_node_status") + test_modify_pod_status || failed+=("modify_pod_status") + test_check_node_lease_transitions 0 || failed+=("check_node_lease_transitions") + + recreate_kwok || failed+=("recreate_kwok") + recreate_pods || failed+=("recreate_pods") + test_node_ready || failed+=("node_ready_again") + test_pod_running || failed+=("pod_running_again") + test_check_pod_status || failed+=("check_pod_status_again") + + test_check_node_lease_transitions 1 || failed+=("check_node_lease_transitions_again") + + if [[ "${#failed[@]}" -ne 0 ]]; then + echo "Error: Some tests failed" + for test in "${failed[@]}"; do + echo " - ${test}" + done + exit 1 + fi +} + +main diff --git a/test/kwok-with-cni/kustomization.yaml b/test/kwok-with-cni/kustomization.yaml index 57bb52c570..fc43bf75df 100644 --- a/test/kwok-with-cni/kustomization.yaml +++ b/test/kwok-with-cni/kustomization.yaml @@ -2,11 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - - name: registry.k8s.io/kwok/kwok - newName: kwok-with-cni - newTag: test +- name: registry.k8s.io/kwok/kwok + newName: kwok-with-cni + newTag: test resources: - - ../../kustomize/kwok-with-cni - - fake-deployment.yaml - - fake-node.yaml +- ../../kustomize/kwok-with-cni +- fake-deployment.yaml +- fake-node.yaml diff --git a/test/kwok/kustomization.yaml b/test/kwok/kustomization.yaml index cfba275082..cee80363a1 100644 --- a/test/kwok/kustomization.yaml +++ b/test/kwok/kustomization.yaml @@ -2,9 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - - name: registry.k8s.io/kwok/kwok - newName: kwok - newTag: test +- name: registry.k8s.io/kwok/kwok + newName: kwok + newTag: test patches: - target: