diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2a508f7dbb..985117e47b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,6 +75,7 @@ jobs: case: - kwok - kwok-with-cni + - kwok-single-node continue-on-error: false runs-on: ubuntu-latest steps: diff --git a/kustomize/kwok/deployment.yaml b/kustomize/kwok/deployment.yaml index 315b96b2cf..ab7193ebbd 100644 --- a/kustomize/kwok/deployment.yaml +++ b/kustomize/kwok/deployment.yaml @@ -14,6 +14,7 @@ spec: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - --manage-nodes-with-label-selector= + - --manage-single-node= - --disregard-status-with-annotation-selector=kwok.x-k8s.io/status=custom - --disregard-status-with-label-selector= - --node-ip=$(POD_IP) diff --git a/pkg/apis/config/v1alpha1/kwok_configuration_types.go b/pkg/apis/config/v1alpha1/kwok_configuration_types.go index 3eb985c803..633c95945e 100644 --- a/pkg/apis/config/v1alpha1/kwok_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwok_configuration_types.go @@ -70,19 +70,33 @@ type KwokConfigurationOptions struct { // is the default value for flag --tls-private-key-file TLSPrivateKeyFile string `json:"tlsPrivateKeyFile,omitempty"` + // ManageSingleNode is the option to manage a single node name. + // is the default value for flag --manage-single-node + // Note: when `manage-all-nodes` is specified as true or + // `manage-nodes-with-label-selector` or `manage-nodes-with-annotation-selector` is specified, + // this is a no-op. + ManageSingleNode string `json:"manageSingleNode,omitempty"` + // Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. // is the default value for flag --manage-all-nodes + // Note: when `manage-single-node` is specified as true or + // `manage-nodes-with-label-selector` or `manage-nodes-with-annotation-selector` is specified, + // this is a no-op. // +default=false ManageAllNodes *bool `json:"manageAllNodes,omitempty"` // Default annotations specified on Nodes to demand manage. - // Note: when `all-node-manage` is specified as true, this is a no-op. // is the default value for flag --manage-nodes-with-annotation-selector + // Note: when `all-node-manage` is specified as true or + // `manage-single-node` is specified, + // this is a no-op. ManageNodesWithAnnotationSelector string `json:"manageNodesWithAnnotationSelector,omitempty"` // Default labels specified on Nodes to demand manage. - // Note: when `all-node-manage` is specified as true, this is a no-op. // is the default value for flag --manage-nodes-with-label-selector + // Note: when `all-node-manage` is specified as true or + // `manage-single-node` is specified, + // this is a no-op. ManageNodesWithLabelSelector string `json:"manageNodesWithLabelSelector,omitempty"` // If a Node/Pod is on a managed Node and has this annotation status will not be modified diff --git a/pkg/apis/internalversion/kwok_configuration_types.go b/pkg/apis/internalversion/kwok_configuration_types.go index e100a09cdc..57b8729f9e 100644 --- a/pkg/apis/internalversion/kwok_configuration_types.go +++ b/pkg/apis/internalversion/kwok_configuration_types.go @@ -53,6 +53,9 @@ type KwokConfigurationOptions struct { // TLSPrivateKeyFile is the ile containing x509 private key TLSPrivateKeyFile string + // ManageSingleNode is the option to manage a single node name + ManageSingleNode string + // Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. ManageAllNodes bool @@ -74,11 +77,11 @@ type KwokConfigurationOptions struct { // Experimental support for getting pod ip from CNI, for CNI-related components, Only works with Linux. EnableCNI bool - // enableDebuggingHandlers enables server endpoints for log collection + // EnableDebuggingHandlers enables server endpoints for log collection // and local running of containers and commands EnableDebuggingHandlers bool - // enableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. + // EnableContentionProfiling enables lock contention profiling, if enableDebuggingHandlers is true. EnableContentionProfiling bool // EnableProfiling enables /debug/pprof handler. diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 92b9ff6161..50bede41b2 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1231,6 +1231,7 @@ func autoConvert_internalversion_KwokConfigurationOptions_To_v1alpha1_KwokConfig out.NodePort = in.NodePort out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.ManageSingleNode = in.ManageSingleNode if err := v1.Convert_bool_To_Pointer_bool(&in.ManageAllNodes, &out.ManageAllNodes, s); err != nil { return err } @@ -1271,6 +1272,7 @@ func autoConvert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfig out.NodePort = in.NodePort out.TLSCertFile = in.TLSCertFile out.TLSPrivateKeyFile = in.TLSPrivateKeyFile + out.ManageSingleNode = in.ManageSingleNode if err := v1.Convert_Pointer_bool_To_bool(&in.ManageAllNodes, &out.ManageAllNodes, s); err != nil { return err } diff --git a/pkg/kwok/cmd/root.go b/pkg/kwok/cmd/root.go index d88d916a87..2720ee669b 100644 --- a/pkg/kwok/cmd/root.go +++ b/pkg/kwok/cmd/root.go @@ -79,9 +79,10 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().IntVar(&flags.Options.NodePort, "node-port", flags.Options.NodePort, "Port of the node") cmd.Flags().StringVar(&flags.Options.TLSCertFile, "tls-cert-file", flags.Options.TLSCertFile, "File containing the default x509 Certificate for HTTPS") cmd.Flags().StringVar(&flags.Options.TLSPrivateKeyFile, "tls-private-key-file", flags.Options.TLSPrivateKeyFile, "File containing the default x509 private key matching --tls-cert-file") - cmd.Flags().BoolVar(&flags.Options.ManageAllNodes, "manage-all-nodes", flags.Options.ManageAllNodes, "All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector and manage-nodes-with-label-selector.") - cmd.Flags().StringVar(&flags.Options.ManageNodesWithAnnotationSelector, "manage-nodes-with-annotation-selector", flags.Options.ManageNodesWithAnnotationSelector, "Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes.") - cmd.Flags().StringVar(&flags.Options.ManageNodesWithLabelSelector, "manage-nodes-with-label-selector", flags.Options.ManageNodesWithLabelSelector, "Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes.") + cmd.Flags().StringVar(&flags.Options.ManageSingleNode, "manage-single-node", flags.Options.ManageSingleNode, "Node that matches the name will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-all-nodes.") + cmd.Flags().BoolVar(&flags.Options.ManageAllNodes, "manage-all-nodes", flags.Options.ManageAllNodes, "All nodes will be watched and managed. It's conflicted with manage-nodes-with-annotation-selector, manage-nodes-with-label-selector and manage-single-node.") + cmd.Flags().StringVar(&flags.Options.ManageNodesWithAnnotationSelector, "manage-nodes-with-annotation-selector", flags.Options.ManageNodesWithAnnotationSelector, "Nodes that match the annotation selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node.") + cmd.Flags().StringVar(&flags.Options.ManageNodesWithLabelSelector, "manage-nodes-with-label-selector", flags.Options.ManageNodesWithLabelSelector, "Nodes that match the label selector will be watched and managed. It's conflicted with manage-all-nodes and manage-single-node.") cmd.Flags().StringVar(&flags.Options.DisregardStatusWithAnnotationSelector, "disregard-status-with-annotation-selector", flags.Options.DisregardStatusWithAnnotationSelector, "All node/pod status excluding the ones that match the annotation selector will be watched and managed.") cmd.Flags().StringVar(&flags.Options.DisregardStatusWithLabelSelector, "disregard-status-with-label-selector", flags.Options.DisregardStatusWithLabelSelector, "All node/pod status excluding the ones that match the label selector will be watched and managed.") cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "Path to the kubeconfig file to use") @@ -229,24 +230,6 @@ func runE(ctx context.Context, flags *flagpole) error { return err } - if flags.Options.ManageAllNodes { - if flags.Options.ManageNodesWithAnnotationSelector != "" || flags.Options.ManageNodesWithLabelSelector != "" { - logger.Error("manage-all-nodes is conflicted with manage-nodes-with-annotation-selector and manage-nodes-with-label-selector.", nil) - os.Exit(1) - } - logger.Info("Watch all nodes") - } else if flags.Options.ManageNodesWithAnnotationSelector != "" || flags.Options.ManageNodesWithLabelSelector != "" { - logger.Info("Watch nodes", - "annotation", flags.Options.ManageNodesWithAnnotationSelector, - "label", flags.Options.ManageNodesWithLabelSelector, - ) - } - - err = waitForReady(ctx, typedClient) - if err != nil { - return err - } - id, err := controllers.Identity() if err != nil { return err @@ -261,6 +244,7 @@ func runE(ctx context.Context, flags *flagpole) error { EnableCNI: flags.Options.EnableCNI, EnableMetrics: enableMetrics, EnablePodCache: enableMetrics, + ManageSingleNode: flags.Options.ManageSingleNode, ManageAllNodes: flags.Options.ManageAllNodes, ManageNodesWithAnnotationSelector: flags.Options.ManageNodesWithAnnotationSelector, ManageNodesWithLabelSelector: flags.Options.ManageNodesWithLabelSelector, @@ -282,6 +266,11 @@ func runE(ctx context.Context, flags *flagpole) error { return err } + err = waitForReady(ctx, typedClient) + if err != nil { + return err + } + serverAddress := flags.Options.ServerAddress if serverAddress == "" && flags.Options.NodePort != 0 { serverAddress = "0.0.0.0:" + format.String(flags.Options.NodePort) diff --git a/pkg/kwok/controllers/controller.go b/pkg/kwok/controllers/controller.go index e988938915..966273bfb6 100644 --- a/pkg/kwok/controllers/controller.go +++ b/pkg/kwok/controllers/controller.go @@ -112,6 +112,7 @@ type Config struct { EnableCNI bool TypedClient kubernetes.Interface TypedKwokClient versioned.Interface + ManageSingleNode string ManageAllNodes bool ManageNodesWithAnnotationSelector string ManageNodesWithLabelSelector string @@ -132,16 +133,31 @@ type Config struct { EnablePodCache bool } -// NewController creates a new fake kubelet controller -func NewController(conf Config) (*Controller, error) { +func (c Config) validate() error { switch { - case conf.ManageAllNodes: - conf.ManageNodesWithAnnotationSelector = "" - conf.ManageNodesWithLabelSelector = "" - case conf.ManageNodesWithAnnotationSelector != "": - case conf.ManageNodesWithLabelSelector != "": + case c.ManageSingleNode != "": + if c.ManageAllNodes { + return fmt.Errorf("manage-single-node is conflicted with manage-all-nodes") + } + if c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "" { + return fmt.Errorf("manage-single-node is conflicted with manage-nodes-with-annotation-selector or manage-nodes-with-label-selector") + } + case c.ManageAllNodes: + if c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "" { + return fmt.Errorf("manage-all-nodes is conflicted with manage-nodes-with-annotation-selector or manage-nodes-with-label-selector") + } + case c.ManageNodesWithAnnotationSelector != "" || c.ManageNodesWithLabelSelector != "": default: - return nil, fmt.Errorf("no nodes are managed") + return fmt.Errorf("no nodes are managed") + } + return nil +} + +// NewController creates a new fake kubelet controller +func NewController(conf Config) (*Controller, error) { + err := conf.validate() + if err != nil { + return nil, err } n := &Controller{ @@ -170,14 +186,34 @@ func (c *Controller) Start(ctx context.Context) error { onLeaseNodeManageFunc func(nodeName string) onNodeManagedFunc func(nodeName string) readOnlyFunc func(nodeName string) bool + + manageNodesWithLabelSelector string + manageNodesWithAnnotationSelector string + manageNodesWithFieldSelector string + manageNodeLeasesWithFieldSelector string + managePodsWithFieldSelector string ) + switch { + case conf.ManageSingleNode != "": + managePodsWithFieldSelector = fields.OneTermEqualSelector("spec.nodeName", conf.ManageSingleNode).String() + manageNodesWithFieldSelector = fields.OneTermEqualSelector("metadata.name", conf.ManageSingleNode).String() + manageNodeLeasesWithFieldSelector = fields.OneTermEqualSelector("metadata.name", conf.ManageSingleNode).String() + case conf.ManageAllNodes: + managePodsWithFieldSelector = fields.OneTermNotEqualSelector("spec.nodeName", "").String() + case conf.ManageNodesWithLabelSelector != "" || conf.ManageNodesWithAnnotationSelector != "": + manageNodesWithLabelSelector = conf.ManageNodesWithLabelSelector + manageNodesWithAnnotationSelector = conf.ManageNodesWithAnnotationSelector + managePodsWithFieldSelector = fields.OneTermNotEqualSelector("spec.nodeName", "").String() + } + nodeChan := make(chan informer.Event[*corev1.Node], 1) nodesCli := conf.TypedClient.CoreV1().Nodes() nodesInformer := informer.NewInformer[*corev1.Node, *corev1.NodeList](nodesCli) nodesCache, err := nodesInformer.WatchWithCache(ctx, informer.Option{ - LabelSelector: conf.ManageNodesWithLabelSelector, - AnnotationSelector: conf.ManageNodesWithAnnotationSelector, + LabelSelector: manageNodesWithLabelSelector, + AnnotationSelector: manageNodesWithAnnotationSelector, + FieldSelector: manageNodesWithFieldSelector, }, nodeChan) if err != nil { return fmt.Errorf("failed to watch nodes: %w", err) @@ -188,7 +224,7 @@ func (c *Controller) Start(ctx context.Context) error { podsInformer := informer.NewInformer[*corev1.Pod, *corev1.PodList](podsCli) podWatchOption := informer.Option{ - FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", "").String(), + FieldSelector: managePodsWithFieldSelector, } var podsCache informer.Getter[*corev1.Pod] @@ -205,7 +241,9 @@ func (c *Controller) Start(ctx context.Context) error { nodeLeasesChan = make(chan informer.Event[*coordinationv1.Lease], 1) nodeLeasesCli := conf.TypedClient.CoordinationV1().Leases(corev1.NamespaceNodeLease) nodeLeasesInformer := informer.NewInformer[*coordinationv1.Lease, *coordinationv1.LeaseList](nodeLeasesCli) - err = nodeLeasesInformer.Watch(ctx, informer.Option{}, nodeLeasesChan) + err = nodeLeasesInformer.Watch(ctx, informer.Option{ + FieldSelector: manageNodeLeasesWithFieldSelector, + }, nodeLeasesChan) if err != nil { return fmt.Errorf("failed to watch nodes: %w", err) } diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 0464b24f67..43520013df 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -1860,6 +1860,21 @@ is the default value for flag –tls-private-key-file
manageSingleNode
+
+string
+
+ManageSingleNode is the option to manage a single node name.
+is the default value for flag –manage-single-node
+Note: when manage-all-nodes
is specified as true or
+manage-nodes-with-label-selector
or manage-nodes-with-annotation-selector
is specified,
+this is a no-op.
manageAllNodes
bool
@@ -1867,7 +1882,10 @@ bool
Default option to manage (i.e., maintain heartbeat/liveness of) all Nodes or not. -is the default value for flag –manage-all-nodes
+is the default value for flag –manage-all-nodes +Note: whenmanage-single-node
is specified as true or
+manage-nodes-with-label-selector
or manage-nodes-with-annotation-selector
is specified,
+this is a no-op.
Default annotations specified on Nodes to demand manage.
-Note: when all-node-manage
is specified as true, this is a no-op.
-is the default value for flag –manage-nodes-with-annotation-selector
all-node-manage
is specified as true or
+manage-single-node
is specified,
+this is a no-op.
Default labels specified on Nodes to demand manage.
-Note: when all-node-manage
is specified as true, this is a no-op.
-is the default value for flag –manage-nodes-with-label-selector
all-node-manage
is specified as true or
+manage-single-node
is specified,
+this is a no-op.