diff --git a/examples/daemonsetsharding/deployment-no-node-pods.yaml b/examples/daemonsetsharding/deployment-no-node-pods.yaml index 3f2df1f81d..73a0c01dfd 100644 --- a/examples/daemonsetsharding/deployment-no-node-pods.yaml +++ b/examples/daemonsetsharding/deployment-no-node-pods.yaml @@ -23,7 +23,7 @@ spec: containers: - args: - --resources=pods - - --node="" + - --enable-no-node-scrape image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.12.0 livenessProbe: httpGet: diff --git a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet index e0439a4df6..a5de0d9718 100644 --- a/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet +++ b/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet @@ -377,7 +377,7 @@ local c = ksm.deployment.spec.template.spec.containers[0] { args: [ '--resources=pods', - '--node=""', + '--enable-no-node-scrape', ], name: shardksmname, }; @@ -410,7 +410,7 @@ local c = ksm.deployment.spec.template.spec.containers[0] { args: [ '--resources=pods', - '--node=""', + '--enable-no-node-scrape', ], }; local shardksmname = ksm.name + "-no-node-pods"; diff --git a/pkg/app/server.go b/pkg/app/server.go index aa5b2c916c..1ad8207f89 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -205,7 +205,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { namespaces := opts.Namespaces.GetNamespaces() nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist) - nodeFieldSelector := opts.Node.GetNodeFieldSelector() + nodeFieldSelector := opts.Node.GetNodeFieldSelector(opts.EnableNoNodeScrape) merged, err := storeBuilder.MergeFieldSelectors([]string{nsFieldSelector, nodeFieldSelector}) if err != nil { return err diff --git a/pkg/options/options.go b/pkg/options/options.go index 09adf8939a..ac2ae45d99 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -46,6 +46,7 @@ type Options struct { Namespaces NamespaceList `yaml:"namespaces"` NamespacesDenylist NamespaceList `yaml:"namespaces_denylist"` Node NodeType `yaml:"node"` + EnableNoNodeScrape bool `yaml:"enable_no_node_scrape"` Pod string `yaml:"pod"` Port int `yaml:"port"` Resources ResourceSet `yaml:"resources"` @@ -73,7 +74,6 @@ func NewOptions() *Options { MetricAllowlist: MetricSet{}, MetricDenylist: MetricSet{}, MetricOptInList: MetricSet{}, - Node: NodeType{}, AnnotationsAllowList: LabelsAllowList{}, LabelsAllowList: LabelsAllowList{}, } @@ -121,6 +121,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().BoolVar(&o.CustomResourcesOnly, "custom-resource-state-only", false, "Only provide Custom Resource State metrics (experimental)") o.cmd.Flags().BoolVar(&o.EnableGZIPEncoding, "enable-gzip-encoding", false, "Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.") + o.cmd.Flags().BoolVar(&o.EnableNoNodeScrape, "enable-no-node-scrape", false, "This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of no scheduled pods is scraped. This is experimental.") o.cmd.Flags().BoolVarP(&o.Help, "help", "h", false, "Print Help text") o.cmd.Flags().BoolVarP(&o.UseAPIServerCache, "use-apiserver-cache", "", false, "Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.") o.cmd.Flags().Int32Var(&o.Shard, "shard", int32(0), "The instances shard nominal (zero indexed) within the total number of shards. (default 0)") @@ -137,7 +138,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { o.cmd.Flags().StringVar(&o.TLSConfig, "tls-config", "", "Path to the TLS configuration file") o.cmd.Flags().StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`) o.cmd.Flags().StringVar(&o.Config, "config", "", "Path to the kube-state-metrics options config file") - o.cmd.Flags().Var(&o.Node, "node", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") + o.cmd.Flags().StringVar((*string)(&o.Node), "node", "", "Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.") o.cmd.Flags().Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').") o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.") o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.") @@ -162,7 +163,7 @@ func (o *Options) Usage() { // Validate validates arguments func (o *Options) Validate() error { shardableResource := "pods" - if o.Node.String() == "" { + if o.Node == "" { return nil } for _, x := range o.Resources.AsSlice() { diff --git a/pkg/options/types.go b/pkg/options/types.go index 376ef9b526..61579d810b 100644 --- a/pkg/options/types.go +++ b/pkg/options/types.go @@ -105,61 +105,18 @@ func (r *ResourceSet) Type() string { } // NodeType represents a nodeName to query from. -type NodeType map[string]struct{} - -// Set converts a comma-separated string of nodename into a slice and appends it to the NodeList -func (n *NodeType) Set(value string) error { - s := *n - cols := strings.Split(value, ",") - for _, col := range cols { - col = strings.TrimSpace(col) - if len(col) != 0 { - s[col] = struct{}{} - } - } - return nil -} - -// AsSlice returns the LabelsAllowList in the form of plain string slice. -func (n NodeType) AsSlice() []string { - cols := make([]string, 0, len(n)) - for col := range n { - cols = append(cols, col) - } - return cols -} - -func (n NodeType) String() string { - klog.InfoS("n.AsSlice()", n.AsSlice()) - return strings.Join(n.AsSlice(), ",") -} - -// Type returns a descriptive string about the NodeList type. -func (n *NodeType) Type() string { - return "string" -} +type NodeType string // GetNodeFieldSelector returns a nodename field selector. -func (n *NodeType) GetNodeFieldSelector() string { - if nil == n { - klog.InfoS("Using node type is nil") - return EmptyFieldSelector() - } - nodeName := n.String() - // `--node=""` find pods without node name assigned which uses fieldselector spec.nodeName="" - klog.InfoS("Using node name", nodeName) - if nodeName == "" { +func (n *NodeType) GetNodeFieldSelector(noNodeAssigned bool) string { + if noNodeAssigned { klog.InfoS("Using spec.nodeName= to select unscheduable pods without node") return "spec.nodeName=" } - klog.InfoS("Using spec.nodeName=", nodeName) - return fields.OneTermEqualSelector("spec.nodeName", nodeName).String() - -} - -// NodeValue represents a nodeName to query from. -type NodeValue interface { - GetNodeFieldSelector() string + if string(*n) != "" { + return fields.OneTermEqualSelector("spec.nodeName", string(*n)).String() + } + return EmptyFieldSelector() } // EmptyFieldSelector returns an empty field selector. diff --git a/pkg/options/types_test.go b/pkg/options/types_test.go index 4b89f76f4f..96ec0675bc 100644 --- a/pkg/options/types_test.go +++ b/pkg/options/types_test.go @@ -162,37 +162,38 @@ func TestNodeFieldSelector(t *testing.T) { Wanted string }{ { - Desc: "with node name", + Desc: "empty node name", + Node: "", Wanted: "", }, { Desc: "with node name", - Node: nil, - Wanted: "", - }, - { - Desc: "empty node name", - Node: NodeType( - map[string]struct{}{ - "": {}, - }, - ), - Wanted: "spec.nodeName=", - }, - { - Desc: "with node name", - Node: NodeType( - map[string]struct{}{ - "k8s-node-1": {}, - }, - ), + Node: "k8s-node-1", Wanted: "spec.nodeName=k8s-node-1", }, } for _, test := range tests { node := test.Node - actual := node.GetNodeFieldSelector() + actual := node.GetNodeFieldSelector(false) + if !reflect.DeepEqual(actual, test.Wanted) { + t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual) + } + } + tests1 := []struct { + Desc string + Node NodeType + Wanted string + }{ + { + Desc: "empty node name", + Node: "", + Wanted: "spec.nodeName=", + }, + } + for _, test := range tests1 { + node := test.Node + actual := node.GetNodeFieldSelector(true) if !reflect.DeepEqual(actual, test.Wanted) { t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual) } @@ -211,67 +212,43 @@ func TestMergeFieldSelectors(t *testing.T) { Desc: "empty DeniedNamespaces", Namespaces: NamespaceList{"default", "kube-system"}, DeniedNamespaces: NamespaceList{}, - Node: NodeType( - map[string]struct{}{ - "": {}, - }, - ), - Wanted: "spec.nodeName=", + Node: "", + Wanted: "", }, { Desc: "all DeniedNamespaces", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"some-system"}, - Node: NodeType( - map[string]struct{}{ - "": {}, - }, - ), - Wanted: "metadata.namespace!=some-system,spec.nodeName=", + Node: "", + Wanted: "metadata.namespace!=some-system", }, { Desc: "general case", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, - Node: NodeType( - map[string]struct{}{ - "": {}, - }, - ), - Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=", + Node: "", + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system", }, { Desc: "empty DeniedNamespaces", Namespaces: NamespaceList{"default", "kube-system"}, DeniedNamespaces: NamespaceList{}, - Node: NodeType( - map[string]struct{}{ - "k8s-node-1": {}, - }, - ), - Wanted: "spec.nodeName=k8s-node-1", + Node: "k8s-node-1", + Wanted: "spec.nodeName=k8s-node-1", }, { Desc: "all DeniedNamespaces", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"some-system"}, - Node: NodeType( - map[string]struct{}{ - "k8s-node-1": {}, - }, - ), - Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1", + Node: "k8s-node-1", + Wanted: "metadata.namespace!=some-system,spec.nodeName=k8s-node-1", }, { Desc: "general case", Namespaces: DefaultNamespaces, DeniedNamespaces: NamespaceList{"case1-system", "case2-system"}, - Node: NodeType( - map[string]struct{}{ - "k8s-node-1": {}, - }, - ), - Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1", + Node: "k8s-node-1", + Wanted: "metadata.namespace!=case1-system,metadata.namespace!=case2-system,spec.nodeName=k8s-node-1", }, } @@ -279,7 +256,7 @@ func TestMergeFieldSelectors(t *testing.T) { ns := test.Namespaces deniedNS := test.DeniedNamespaces selector1 := ns.GetExcludeNSFieldSelector(deniedNS) - selector2 := test.Node.GetNodeFieldSelector() + selector2 := test.Node.GetNodeFieldSelector(false) actual, err := MergeFieldSelectors([]string{selector1, selector2}) if err != nil { t.Errorf("Test error for Desc: %s. Can't merge field selector %v.", test.Desc, err) diff --git a/tests/e2e.sh b/tests/e2e.sh index 8a8773d4b0..520782b8be 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -160,6 +160,8 @@ function test_daemonset() { sleep 3 curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-no-node-pods:http-metrics/proxy/metrics" sleep 3 + kubectl get pods -A --field-selector spec.nodeName="\"\"" + sleep 3 kubectl get pods -A --field-selector spec.nodeName="" sleep 3 pendingpod2="$(curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics-no-node-pods:http-metrics/proxy/metrics" | grep "pendingpod2" | grep -c "kube_pod_info" )" @@ -244,6 +246,8 @@ echo "kube-state-metrics is up and running" echo "start e2e test for kube-state-metrics" KSM_HTTP_METRICS_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy' KSM_TELEMETRY_URL='http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:telemetry/proxy' + +kubectl --namespace=kube-system logs deployment/kube-state-metrics kube-state-metrics go test -v ./tests/e2e/main_test.go --ksm-http-metrics-url=${KSM_HTTP_METRICS_URL} --ksm-telemetry-url=${KSM_TELEMETRY_URL} # TODO: re-implement the following test cases in Go with the goal of removing this file. diff --git a/tests/e2e/testdata/pods.yaml b/tests/e2e/testdata/pods.yaml index 523e7b3317..f3b14b21a2 100644 --- a/tests/e2e/testdata/pods.yaml +++ b/tests/e2e/testdata/pods.yaml @@ -26,7 +26,7 @@ spec: - command: - /agnhost - netexec - - --http-port=8080q + - --http-port=8080 image: registry.k8s.io/e2e-test-images/agnhost:2.39 imagePullPolicy: IfNotPresent name: agnhost