Skip to content

Commit

Permalink
Allow optional VK in CR metrics
Browse files Browse the repository at this point in the history
Allow optional VK in CR metrics, while only requiring the group to be
fixed. This would allow users to define custom metrics for:
* a specific API version: version is fixed, kind varies.
* all the API versions of a resource: version varies, kind is fixed.
* all the APIs part of an API group: version varies, kind varies.
* also allow users to keep a tab on CRDs by exposing metrics for the
  same.

Signed-off-by: Pranshu Srivastava <rexagod@gmail.com>
  • Loading branch information
rexagod committed Nov 17, 2022
1 parent 34b4660 commit c9bea21
Show file tree
Hide file tree
Showing 6 changed files with 146 additions and 12 deletions.
6 changes: 6 additions & 0 deletions docs/customresourcedefinition-metrics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
CustomResourceDefinition Metrics

| Metric name | Metric type | Labels/tags | Status |
|---------------------------------------|-------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
| kube_customresourcedefinition_created | Gauge | `customresourcedefinition`=&lt;customresourcedefinition-name&gt; <br> `namespace`=&lt;customresourcedefinition-namespace&gt; | EXPERIMENTAL |
| kube_customresourcedefinition_info | Gauge | `customresourcedefinition`=&lt;customresourcedefinition-name&gt; <br> `namespace`=&lt;customresourcedefinition-namespace&gt; <br> `group`=&lt;customresourcedefinition-group&gt; <br> `versions`=&lt;customresourcedefinition-versions&gt; `kind`=&lt;customresourcedefinition-kind&gt; <br> `scope`=&lt;customresourcedefinition-scope&gt; | EXPERIMENTAL |
9 changes: 5 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ require (
github.com/stretchr/testify v1.8.1
golang.org/x/perf v0.0.0-20220920022801-e8d778a60d07
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.25.3
k8s.io/apimachinery v0.25.3
k8s.io/api v0.25.4
k8s.io/apiextensions-apiserver v0.25.4
k8s.io/apimachinery v0.25.4
k8s.io/autoscaler/vertical-pod-autoscaler v0.12.0
k8s.io/client-go v0.25.3
k8s.io/component-base v0.25.3
k8s.io/client-go v0.25.4
k8s.io/component-base v0.25.4
k8s.io/klog/v2 v2.80.1
k8s.io/sample-controller v0.25.3
k8s.io/utils v0.0.0-20221101230645-61b03e2f6476
Expand Down
18 changes: 10 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -816,16 +816,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ=
k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI=
k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc=
k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs=
k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ=
k8s.io/apiextensions-apiserver v0.25.4 h1:7hu9pF+xikxQuQZ7/30z/qxIPZc2J1lFElPtr7f+B6U=
k8s.io/apiextensions-apiserver v0.25.4/go.mod h1:bkSGki5YBoZWdn5pWtNIdGvDrrsRWlmnvl9a+tAw5vQ=
k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc=
k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo=
k8s.io/autoscaler/vertical-pod-autoscaler v0.12.0 h1:cy8LoXSl5GkTNJDTx3ZCS143f9Ai7gqnGkoUxPlGSmI=
k8s.io/autoscaler/vertical-pod-autoscaler v0.12.0/go.mod h1:LraL5kR2xX7jb4VMCG6/tUH4I75uRHlnzC0VWQHcyWk=
k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0=
k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA=
k8s.io/component-base v0.25.3 h1:UrsxciGdrCY03ULT1h/S/gXFCOPnLhUVwSyx+hM/zq4=
k8s.io/component-base v0.25.3/go.mod h1:WYoS8L+IlTZgU7rhAl5Ctpw0WdMxDfCC5dkxcEFa/TI=
k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8=
k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw=
k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA=
Expand Down
7 changes: 7 additions & 0 deletions internal/store/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
policyv1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1"
storagev1 "k8s.io/api/storage/v1"
apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
vpaautoscaling "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2"
vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
clientset "k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -72,6 +73,7 @@ type Builder struct {
shard int32
totalShards int
buildStoresFunc ksmtypes.BuildStoresFunc
buildCustomStoresFunc ksmtypes.BuildCustomStoresFunc
buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
allowAnnotationsList map[string][]string
allowLabelsList map[string][]string
Expand Down Expand Up @@ -290,6 +292,7 @@ var availableStores = map[string]func(f *Builder) []cache.Store{
"configmaps": func(b *Builder) []cache.Store { return b.buildConfigMapStores() },
"clusterrolebindings": func(b *Builder) []cache.Store { return b.buildClusterRoleBindingStores() },
"cronjobs": func(b *Builder) []cache.Store { return b.buildCronJobStores() },
"customresourcedefinitions": func(b *Builder) []cache.Store { return b.buildCustomResourceDefinitionStores() },
"daemonsets": func(b *Builder) []cache.Store { return b.buildDaemonSetStores() },
"deployments": func(b *Builder) []cache.Store { return b.buildDeploymentStores() },
"endpoints": func(b *Builder) []cache.Store { return b.buildEndpointsStores() },
Expand Down Expand Up @@ -334,6 +337,10 @@ func availableResources() []string {
return c
}

func (b *Builder) buildCustomResourceDefinitionStores() []cache.Store {
return b.buildCustomStoresFunc(customResourceDefinitionMetricFamilies(), &apiext.CustomResourceDefinition{}, createCustomResourceDefinitionListWatch, b.useAPIServerCache)
}

func (b *Builder) buildConfigMapStores() []cache.Store {
return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache)
}
Expand Down
110 changes: 110 additions & 0 deletions internal/store/customresourcedefinition.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
Copyright 2022 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package store

import (
"context"
"strings"

v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
basemetrics "k8s.io/component-base/metrics"

"k8s.io/kube-state-metrics/v2/pkg/metric"
generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
)

func customResourceDefinitionMetricFamilies() []generator.FamilyGenerator {
getCRDVersions := func(crds []v1.CustomResourceDefinitionVersion) string {
var versions []string
for _, crd := range crds {
versions = append(versions, crd.String())
}
return strings.Join(versions, ",")
}
return []generator.FamilyGenerator{
*generator.NewFamilyGeneratorWithStability(
"kube_customresourcedefinition_created",
"Unix creation timestamp",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapCustomResourceDefinitionFunc(func(crd *v1.CustomResourceDefinition) *metric.Family {
var ms []*metric.Metric

if !crd.CreationTimestamp.IsZero() {
ms = append(ms, &metric.Metric{
Value: float64(crd.CreationTimestamp.Unix()),
})
}

return &metric.Family{
Metrics: ms,
}
}),
),
*generator.NewFamilyGeneratorWithStability(
"kube_customresourcedefinition_info",
"Information about customresourcedefinition.",
metric.Gauge,
basemetrics.ALPHA,
"",
wrapCustomResourceDefinitionFunc(func(crd *v1.CustomResourceDefinition) *metric.Family {
return &metric.Family{
Metrics: []*metric.Metric{
{
LabelKeys: []string{"group", "versions", "kinds", "scope"},
LabelValues: []string{crd.Spec.Group, getCRDVersions(crd.Spec.Versions), crd.Spec.Names.Kind, string(crd.Spec.Scope)},
Value: 1,
},
},
}
}),
),
}
}

func wrapCustomResourceDefinitionFunc(f func(*v1.CustomResourceDefinition) *metric.Family) func(interface{}) *metric.Family {
return func(obj interface{}) *metric.Family {
crd := obj.(*v1.CustomResourceDefinition)

metricFamily := f(crd)

for _, m := range metricFamily.Metrics {
m.LabelKeys, m.LabelValues = mergeKeyValues(descDeploymentLabelsDefaultLabels, []string{crd.Namespace, crd.Name}, m.LabelKeys, m.LabelValues)
}

return metricFamily
}
}

func createCustomResourceDefinitionListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher {
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
opts.FieldSelector = fieldSelector
return kubeClient.ApiextensionsV1().CustomResourceDefinitions().List(context.TODO(), opts)
},
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
opts.FieldSelector = fieldSelector
return kubeClient.ApiextensionsV1().CustomResourceDefinitions().Watch(context.TODO(), opts)
},
}
}
8 changes: 8 additions & 0 deletions pkg/builder/types/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"

"github.com/prometheus/client_golang/prometheus"
apiextclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
Expand Down Expand Up @@ -62,6 +63,13 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator,
useAPIServerCache bool,
) []cache.Store

// BuildCustomStoresFunc function signature that is used to return a list of cache.Store
type BuildCustomStoresFunc func(metricFamilies []generator.FamilyGenerator,
expectedType interface{},
listWatchFunc func(kubeClient apiextclientset.Interface, ns string, fieldSelector string) cache.ListerWatcher,
useAPIServerCache bool,
) []cache.Store

// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store
type BuildCustomResourceStoresFunc func(resourceName string,
metricFamilies []generator.FamilyGenerator,
Expand Down

0 comments on commit c9bea21

Please sign in to comment.