From 7130bd0d2b0c5ced03b53835ce5aa2997bc509a4 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Tue, 3 Jan 2023 13:28:09 +0100 Subject: [PATCH 1/4] prefix GVK labels in CustomResourceMonitoring This will prefix the auto-generated GVK labels for CustomResources with customresource_ to make it a bit more clear that these labels got generated. Signed-off-by: Mario Constanti --- docs/customresourcestate-metrics.md | 2 +- pkg/customresourcestate/config.go | 6 +- .../custom_resource_metrics_test.go | 188 +++++++++++++++++- pkg/customresourcestate/registry_factory.go | 6 +- .../registry_factory_test.go | 2 +- 5 files changed, 190 insertions(+), 14 deletions(-) diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md index 5296091ebc..d21fe37b30 100644 --- a/docs/customresourcestate-metrics.md +++ b/docs/customresourcestate-metrics.md @@ -49,7 +49,7 @@ spec: - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers ``` -NOTE: The `group`, `version`, and `kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field. +NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field. ### Examples diff --git a/pkg/customresourcestate/config.go b/pkg/customresourcestate/config.go index 288c1e90b3..4ec21f0560 100644 --- a/pkg/customresourcestate/config.go +++ b/pkg/customresourcestate/config.go @@ -26,6 +26,10 @@ import ( "k8s.io/kube-state-metrics/v2/pkg/customresource" ) +// customResourceState is used to prefix the auto-generated GVK labels as well as an appendix for the metric itself +// if no custom metric name is defined +const customResourceState string = "customresource" + // Metrics is the top level configuration object. type Metrics struct { Spec MetricsSpec `yaml:"spec" json:"spec"` @@ -64,7 +68,7 @@ type Resource struct { func (r Resource) GetMetricNamePrefix() string { p := r.MetricNamePrefix if p == nil { - return "kube_crd" + return "kube_" + customResourceState } return *p } diff --git a/pkg/customresourcestate/custom_resource_metrics_test.go b/pkg/customresourcestate/custom_resource_metrics_test.go index 9bd1a5c536..e62e333c1c 100644 --- a/pkg/customresourcestate/custom_resource_metrics_test.go +++ b/pkg/customresourcestate/custom_resource_metrics_test.go @@ -17,18 +17,87 @@ limitations under the License. package customresourcestate import ( + "encoding/json" + "reflect" "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/pointer" ) func TestNewCustomResourceMetrics(t *testing.T) { + tests := []struct { - r Resource - wantErr bool - name string + r Resource + wantErr bool + wantResult *customResourceMetrics + name string }{ { // https://github.com/kubernetes/kube-state-metrics/issues/1886 - name: "dynamic metric type (not just hardcoded to gauge)", + name: "cr metric with dynamic metric type", + r: Resource{ + GroupVersionKind: GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Labels: Labels{ + LabelsFromPath: map[string][]string{ + "name": {"metadata", "name"}, + }, + CommonLabels: map[string]string{ + "hello": "world", + }, + }, + Metrics: []Generator{ + { + Name: "test_metrics", + Help: "metrics for testing", + Each: Metric{ + Type: MetricTypeInfo, + Info: &MetricInfo{ + MetricMeta: MetricMeta{ + Path: []string{ + "metadata", + "annotations", + }, + }, + LabelFromKey: "test", + }, + }, + }, + }, + }, + wantErr: false, + wantResult: &customResourceMetrics{ + MetricNamePrefix: "kube_customresource", + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "kube_customresource_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, + }, + { + name: "cr metric with custom prefix", r: Resource{ GroupVersionKind: GroupVersionKind{ Group: "apps", @@ -39,6 +108,9 @@ func TestNewCustomResourceMetrics(t *testing.T) { LabelsFromPath: map[string][]string{ "name": {"metadata", "name"}, }, + CommonLabels: map[string]string{ + "hello": "world", + }, }, Metrics: []Generator{ { @@ -58,18 +130,118 @@ func TestNewCustomResourceMetrics(t *testing.T) { }, }, }, + MetricNamePrefix: pointer.String("apps_deployment"), }, wantErr: false, + wantResult: &customResourceMetrics{ + MetricNamePrefix: "apps_deployment", + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "apps_deployment_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, + }, + { + name: "cr metric with custom prefix - expect error", + r: Resource{ + GroupVersionKind: GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + Labels: Labels{ + LabelsFromPath: map[string][]string{ + "name": {"metadata", "name"}, + }, + CommonLabels: map[string]string{ + "hello": "world", + }, + }, + Metrics: []Generator{ + { + Name: "test_metrics", + Help: "metrics for testing", + Each: Metric{ + Type: MetricTypeInfo, + Info: &MetricInfo{ + MetricMeta: MetricMeta{ + Path: []string{ + "metadata", + "annotations", + }, + }, + LabelFromKey: "test", + }, + }, + }, + }, + MetricNamePrefix: pointer.String("apps_deployment"), + }, + wantErr: true, + wantResult: &customResourceMetrics{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + ResourceName: "deployments", + Families: []compiledFamily{ + { + Name: "apps_deployment_test_metrics", + Help: "metrics for testing", + Each: &compiledInfo{}, + Labels: map[string]string{ + "customresource_group": "apps", + "customresource_kind": "Deployment", + "customresource_version": "v1", + "hello": "world", + }, + LabelFromPath: map[string]valuePath{ + "name": mustCompilePath(t, "metadata", "name"), + }, + }, + }, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { v, err := NewCustomResourceMetrics(tt.r) - expectedError := v.(*customResourceMetrics).Families[0].Each.Type() != "info" - if (err != nil) != tt.wantErr || expectedError { - t.Errorf("NewCustomResourceMetrics() error = %v, wantErr %v", err, tt.wantErr) - return + if err != nil { + t.Errorf(err.Error()) + } + + // convert to JSON for easier nil comparison + ttWantJSON, _ := json.Marshal(tt.wantResult) + customResourceMetricJSON, _ := json.Marshal(v.(*customResourceMetrics)) + + if !tt.wantErr && !reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) { + t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr) + t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult) + } + + if tt.wantErr && reflect.DeepEqual(ttWantJSON, customResourceMetricJSON) { + t.Errorf("NewCustomResourceMetrics: error expected %v", tt.wantErr) + t.Errorf("NewCustomResourceMetrics:\n %#v\n is not deep equal\n %#v", v, tt.wantResult) } }) } diff --git a/pkg/customresourcestate/registry_factory.go b/pkg/customresourcestate/registry_factory.go index e027cae8e7..7e9eca535c 100644 --- a/pkg/customresourcestate/registry_factory.go +++ b/pkg/customresourcestate/registry_factory.go @@ -38,9 +38,9 @@ func compile(resource Resource) ([]compiledFamily, error) { if resource.CommonLabels == nil { resource.CommonLabels = map[string]string{} } - resource.CommonLabels["group"] = resource.GroupVersionKind.Group - resource.CommonLabels["version"] = resource.GroupVersionKind.Version - resource.CommonLabels["kind"] = resource.GroupVersionKind.Kind + resource.CommonLabels[customResourceState+"_group"] = resource.GroupVersionKind.Group + resource.CommonLabels[customResourceState+"_version"] = resource.GroupVersionKind.Version + resource.CommonLabels[customResourceState+"_kind"] = resource.GroupVersionKind.Kind for _, f := range resource.Metrics { family, err := compileFamily(f, resource) if err != nil { diff --git a/pkg/customresourcestate/registry_factory_test.go b/pkg/customresourcestate/registry_factory_test.go index e4da7d997c..1b42a57029 100644 --- a/pkg/customresourcestate/registry_factory_test.go +++ b/pkg/customresourcestate/registry_factory_test.go @@ -333,7 +333,7 @@ func Test_fullName(t *testing.T) { resource: r(nil), f: count, }, - want: "kube_crd_count", + want: "kube_customresource_count", }, { name: "no prefix", From b4101664715315d7b117663427121775057ed3b6 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Tue, 10 Jan 2023 13:20:08 +0100 Subject: [PATCH 2/4] replace kube_crd with kube_customresource in docs Signed-off-by: Mario Constanti --- docs/customresourcestate-metrics.md | 18 +++++++++--------- docs/verticalpodautoscaler-metrics.md | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md index d21fe37b30..c35db30a94 100644 --- a/docs/customresourcestate-metrics.md +++ b/docs/customresourcestate-metrics.md @@ -117,7 +117,7 @@ spec: Produces the metric: ```prometheus -kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21 +kube_customresource_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 ``` #### Multiple Metrics/Kitchen Sink @@ -169,8 +169,8 @@ spec: Produces the following metrics: ```prometheus -kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2 -kube_crd_ready_count{group="myteam.io", kind="Foo", version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4 +kube_customresource_ready_count{customresource_group="myteam.io", kind="Foo", version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2 +kube_customresource_ready_count{customresource_group="myteam.io", kind="Foo", version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4 ``` ### Metric types @@ -205,7 +205,7 @@ spec: Produces the metric: ```prometheus -kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21 +kube_customresource_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 ``` #### StateSet @@ -237,9 +237,9 @@ The value will be 1, if the value matches the one in list. Produces the metric: ```prometheus -kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Pending"} 1 -kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Bar"} 0 -kube_crd_status_phase{group="myteam.io", kind="Foo", version="v1", phase="Baz"} 0 +kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Pending"} 1 +kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Bar"} 0 +kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Baz"} 0 ``` #### Info @@ -269,7 +269,7 @@ spec: Produces the metric: ```prometheus -kube_crd_version{group="myteam.io", kind="Foo", version="v1", version="v1.2.3"} 1 +kube_customresource_version{customresource_group="myteam.io", kind="Foo", version="v1", version="v1.2.3"} 1 ``` ### Naming @@ -291,7 +291,7 @@ spec: Produces: ```prometheus -myteam_foos_uptime{group="myteam.io", kind="Foo", version="v1"} 43.21 +myteam_foos_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 ``` To omit namespace and/or subsystem altogether, set them to the empty string: diff --git a/docs/verticalpodautoscaler-metrics.md b/docs/verticalpodautoscaler-metrics.md index 49d380de99..75d60bc5c6 100644 --- a/docs/verticalpodautoscaler-metrics.md +++ b/docs/verticalpodautoscaler-metrics.md @@ -30,9 +30,9 @@ spec: gauge: path: [metadata, annotations] # This will output the following metric: -# HELP kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations Kubernetes annotations converted to Prometheus labels. -# TYPE kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations gauge -# kube_crd_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations{namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123 +# HELP kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations Kubernetes annotations converted to Prometheus labels. +# TYPE kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations gauge +# kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations{namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123 ``` PS. The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`). *** From 771f85c457fc788b153c549edda8b2d01300ed45 Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Tue, 10 Jan 2023 13:50:43 +0100 Subject: [PATCH 3/4] fix vpa crd metric names --- docs/verticalpodautoscaler-metrics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/verticalpodautoscaler-metrics.md b/docs/verticalpodautoscaler-metrics.md index 75d60bc5c6..e715af7e88 100644 --- a/docs/verticalpodautoscaler-metrics.md +++ b/docs/verticalpodautoscaler-metrics.md @@ -30,9 +30,9 @@ spec: gauge: path: [metadata, annotations] # This will output the following metric: -# HELP kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations Kubernetes annotations converted to Prometheus labels. -# TYPE kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations gauge -# kube_customresource_autoscaling_k8s_io_v1_VerticalPodAutoscaler_annotations{namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123 +# HELP kube_customresource_autoscaling_annotations Kubernetes annotations converted to Prometheus labels. +# TYPE kube_customresource_autoscaling_annotations gauge +# kube_customresource_autoscaling_annotations{customresource_group="autoscaling.k8s.io", customresource_kind="VerticalPodAutoscaler", customresource_version="v1", namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123 ``` PS. The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`). *** From 488c0bcf590e3992799557eb0db4ea99e8d407ca Mon Sep 17 00:00:00 2001 From: Mario Constanti Date: Tue, 10 Jan 2023 13:51:04 +0100 Subject: [PATCH 4/4] update crd monitoring and mention new flags --- docs/customresourcestate-metrics.md | 54 +++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/docs/customresourcestate-metrics.md b/docs/customresourcestate-metrics.md index c35db30a94..e94e0cddf0 100644 --- a/docs/customresourcestate-metrics.md +++ b/docs/customresourcestate-metrics.md @@ -49,6 +49,40 @@ spec: - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,foos,horizontalpodautoscalers,ingresses,jobs,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments,verticalpodautoscalers ``` +It's also possible to configure kube-state-metrics to run in a `custom-resource-mode` only. In addition to specifying one of `--custom-resource-state-config*` flags, you could set `--custom-resource-state-only` to `true`. +With this configuration only the known custom resources configured in `--custom-resource-state-config*` will be taken into account by kube-state-metrics. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: kube-system +spec: + template: + spec: + containers: + - name: kube-state-metrics + args: + - --custom-resource-state-config + # in YAML files, | allows a multi-line string to be passed as a flag value + # see https://yaml-multiline.info + - | + spec: + resources: + - groupVersionKind: + group: myteam.io + version: "v1" + kind: Foo + metrics: + - name: active_count + help: "Count of active Foo" + each: + type: Gauge + ... + - --custom-resource-state-only=true +``` + NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field. ### Examples @@ -117,7 +151,7 @@ spec: Produces the metric: ```prometheus -kube_customresource_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 +kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 ``` #### Multiple Metrics/Kitchen Sink @@ -169,8 +203,8 @@ spec: Produces the following metrics: ```prometheus -kube_customresource_ready_count{customresource_group="myteam.io", kind="Foo", version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2 -kube_customresource_ready_count{customresource_group="myteam.io", kind="Foo", version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4 +kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a"} 2 +kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b"} 4 ``` ### Metric types @@ -205,7 +239,7 @@ spec: Produces the metric: ```prometheus -kube_customresource_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 +kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 ``` #### StateSet @@ -237,9 +271,9 @@ The value will be 1, if the value matches the one in list. Produces the metric: ```prometheus -kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Pending"} 1 -kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Bar"} 0 -kube_customresource_status_phase{customresource_group="myteam.io", kind="Foo", version="v1", phase="Baz"} 0 +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Pending"} 1 +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Bar"} 0 +kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Baz"} 0 ``` #### Info @@ -269,7 +303,7 @@ spec: Produces the metric: ```prometheus -kube_customresource_version{customresource_group="myteam.io", kind="Foo", version="v1", version="v1.2.3"} 1 +kube_customresource_version{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", version="v1.2.3"} 1 ``` ### Naming @@ -291,7 +325,7 @@ spec: Produces: ```prometheus -myteam_foos_uptime{customresource_group="myteam.io", kind="Foo", version="v1"} 43.21 +myteam_foos_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 ``` To omit namespace and/or subsystem altogether, set them to the empty string: @@ -309,7 +343,7 @@ spec: Produces: ```prometheus -uptime{group="myteam.io", kind="Foo", version="v1"} 43.21 +uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21 ``` ### Logging