diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dda5a917291..70ae25659e6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -115,6 +115,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709] - Add docker `event` metricset. {pull}9856[9856] - Add 'performance' metricset to x-pack mssql module {pull}9826[9826] +- Add DeDot for kubernetes labels and annotations. {issue}9860[9860] {pull}9939[9939] *Packetbeat* diff --git a/libbeat/common/kubernetes/metadata.go b/libbeat/common/kubernetes/metadata.go index fc153d432e3..391d617a8f9 100644 --- a/libbeat/common/kubernetes/metadata.go +++ b/libbeat/common/kubernetes/metadata.go @@ -44,6 +44,8 @@ type MetaGeneratorConfig struct { // Undocumented settings, to be deprecated in favor of `drop_fields` processor: IncludeCreatorMetadata bool `config:"include_creator_metadata"` + LabelsDedot bool `config:"labels.dedot"` + AnnotationsDedot bool `config:"annotations.dedot"` } type metaGenerator = MetaGeneratorConfig @@ -53,6 +55,8 @@ func NewMetaGenerator(cfg *common.Config) (MetaGenerator, error) { // default settings: generator := metaGenerator{ IncludeCreatorMetadata: true, + LabelsDedot: false, + AnnotationsDedot: false, } err := cfg.Unpack(&generator) @@ -70,10 +74,15 @@ func (g *metaGenerator) ResourceMetadata(obj Resource) common.MapStr { labelMap := common.MapStr{} if len(g.IncludeLabels) == 0 { for k, v := range obj.GetMetadata().Labels { - safemapstr.Put(labelMap, k, v) + if g.LabelsDedot { + label := common.DeDot(k) + labelMap.Put(label, v) + } else { + safemapstr.Put(labelMap, k, v) + } } } else { - labelMap = generateMapSubset(objMeta.Labels, g.IncludeLabels) + labelMap = generateMapSubset(objMeta.Labels, g.IncludeLabels, g.LabelsDedot) } // Exclude any labels that are present in the exclude_labels config @@ -81,7 +90,7 @@ func (g *metaGenerator) ResourceMetadata(obj Resource) common.MapStr { delete(labelMap, label) } - annotationsMap := generateMapSubset(objMeta.Annotations, g.IncludeAnnotations) + annotationsMap := generateMapSubset(objMeta.Annotations, g.IncludeAnnotations, g.AnnotationsDedot) meta := common.MapStr{} if objMeta.GetNamespace() != "" { meta["namespace"] = objMeta.GetNamespace() @@ -136,7 +145,7 @@ func (g *metaGenerator) ContainerMetadata(pod *Pod, container string) common.Map return podMeta } -func generateMapSubset(input map[string]string, keys []string) common.MapStr { +func generateMapSubset(input map[string]string, keys []string, dedot bool) common.MapStr { output := common.MapStr{} if input == nil { return output @@ -145,7 +154,12 @@ func generateMapSubset(input map[string]string, keys []string) common.MapStr { for _, key := range keys { value, ok := input[key] if ok { - safemapstr.Put(output, key, value) + if dedot { + dedotKey := common.DeDot(key) + output.Put(dedotKey, value) + } else { + safemapstr.Put(output, key, value) + } } } diff --git a/libbeat/common/kubernetes/metadata_test.go b/libbeat/common/kubernetes/metadata_test.go index 60b6001994e..a4717eb2fe4 100644 --- a/libbeat/common/kubernetes/metadata_test.go +++ b/libbeat/common/kubernetes/metadata_test.go @@ -27,7 +27,7 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TestPodMetadataDeDot(t *testing.T) { +func TestPodMetadata(t *testing.T) { UID := "005f3b90-4b9d-12f8-acf0-31020a840133" Deployment := "Deployment" test := "test" @@ -104,3 +104,88 @@ func TestPodMetadataDeDot(t *testing.T) { assert.Equal(t, metaGen.PodMetadata(test.pod), test.meta) } } + +func TestPodMetadataDeDot(t *testing.T) { + UID := "005f3b90-4b9d-12f8-acf0-31020a840133" + Deployment := "Deployment" + test := "test" + ReplicaSet := "ReplicaSet" + True := true + False := false + tests := []struct { + pod *Pod + meta common.MapStr + config *common.Config + }{ + { + pod: &Pod{ + Metadata: &metav1.ObjectMeta{ + Labels: map[string]string{"a.key": "foo", "a": "bar"}, + Uid: &UID, + Namespace: &test, + Annotations: map[string]string{"b.key": "foo", "b": "bar"}, + }, + Spec: &v1.PodSpec{ + NodeName: &test, + }, + }, + meta: common.MapStr{ + "pod": common.MapStr{ + "name": "", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{"name": "test"}, + "namespace": "test", + "labels": common.MapStr{"a": "bar", "a_key": "foo"}, + "annotations": common.MapStr{"b": "bar", "b_key": "foo"}, + }, + config: common.NewConfig(), + }, + { + pod: &Pod{ + Metadata: &metav1.ObjectMeta{ + Labels: map[string]string{"a.key": "foo", "a": "bar"}, + Uid: &UID, + OwnerReferences: []*metav1.OwnerReference{ + { + Kind: &Deployment, + Name: &test, + Controller: &True, + }, + { + Kind: &ReplicaSet, + Name: &ReplicaSet, + Controller: &False, + }, + }, + }, + Spec: &v1.PodSpec{ + NodeName: &test, + }, + }, + meta: common.MapStr{ + "pod": common.MapStr{ + "name": "", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{"name": "test"}, + "labels": common.MapStr{"a": "bar", "a_key": "foo"}, + "deployment": common.MapStr{"name": "test"}, + }, + config: common.NewConfig(), + }, + } + + for _, test := range tests { + config, err := common.NewConfigFrom(map[string]interface{}{ + "labels.dedot": true, + "annotations.dedot": true, + "include_annotations": []string{"b", "b.key"}, + }) + metaGen, err := NewMetaGenerator(config) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, metaGen.PodMetadata(test.pod), test.meta) + } +} diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index cc3ce48197a..25b65323e69 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -130,6 +130,18 @@ event: If the `include_annotations` config is added to the provider config, then the list of annotations present in the config are added to the event. +If the `include_labels` config is added to the provider config, then the list of labels present in the config +will be added to the event. + +If the `exclude_labels` config is added to the provider config, then the list of labels present in the config +will be excluded from the event. + +if the `labels.dedot` config is set to be `true` in the provider config, then `.` in labels will be replaced with `_`. + +if the `annotations.dedot` config is set to be `true` in the provider config, then `.` in annotations will be replaced +with `_`. + + For example: [source,yaml] diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 82288c62012..ec678aa79b5 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -17,6 +17,8 @@ # Enriching parameters: #add_metadata: true #in_cluster: true + #labels.dedot: false + #annotations.dedot: false # When used outside the cluster: #in_cluster: false #host: node_name diff --git a/metricbeat/module/kubernetes/event/config.go b/metricbeat/module/kubernetes/event/config.go index d2117634028..daabe2b292e 100644 --- a/metricbeat/module/kubernetes/event/config.go +++ b/metricbeat/module/kubernetes/event/config.go @@ -23,10 +23,12 @@ import ( ) type kubeEventsConfig struct { - InCluster bool `config:"in_cluster"` - KubeConfig string `config:"kube_config"` - Namespace string `config:"namespace"` - SyncPeriod time.Duration `config:"sync_period"` + InCluster bool `config:"in_cluster"` + KubeConfig string `config:"kube_config"` + Namespace string `config:"namespace"` + SyncPeriod time.Duration `config:"sync_period"` + LabelsDedot bool `config:"labels.dedot"` + AnnotationsDedot bool `config:"annotations.dedot"` } type Enabled struct { @@ -35,8 +37,10 @@ type Enabled struct { func defaultKubernetesEventsConfig() kubeEventsConfig { return kubeEventsConfig{ - InCluster: true, - SyncPeriod: 1 * time.Second, + InCluster: true, + SyncPeriod: 1 * time.Second, + LabelsDedot: false, + AnnotationsDedot: false, } } diff --git a/metricbeat/module/kubernetes/event/event.go b/metricbeat/module/kubernetes/event/event.go index 6aeab8996f2..a6813624eed 100644 --- a/metricbeat/module/kubernetes/event/event.go +++ b/metricbeat/module/kubernetes/event/event.go @@ -41,7 +41,16 @@ func init() { // MetricSet implements the mb.PushMetricSet interface, and therefore does not rely on polling. type MetricSet struct { mb.BaseMetricSet - watcher kubernetes.Watcher + watcher kubernetes.Watcher + watchOptions kubernetes.WatchOptions + dedotConfig dedotConfig +} + +// dedotConfig defines LabelsDedot and AnnotationsDedot. +// Default to be false. If set to true, replace dots in labels with `_`. +type dedotConfig struct { + LabelsDedot bool `config:"labels.dedot"` + AnnotationsDedot bool `config:"annotations.dedot"` } // New create a new instance of the MetricSet @@ -62,17 +71,26 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, fmt.Errorf("fail to get kubernetes client: %s", err.Error()) } - watcher, err := kubernetes.NewWatcher(client, &kubernetes.Event{}, kubernetes.WatchOptions{ + watchOptions := kubernetes.WatchOptions{ SyncTimeout: config.SyncPeriod, Namespace: config.Namespace, - }) + } + + watcher, err := kubernetes.NewWatcher(client, &kubernetes.Event{}, watchOptions) if err != nil { return nil, fmt.Errorf("fail to init kubernetes watcher: %s", err.Error()) } + dedotConfig := dedotConfig{ + LabelsDedot: config.LabelsDedot, + AnnotationsDedot: config.AnnotationsDedot, + } + return &MetricSet{ BaseMetricSet: base, + dedotConfig: dedotConfig, watcher: watcher, + watchOptions: watchOptions, }, nil } @@ -81,10 +99,10 @@ func (m *MetricSet) Run(reporter mb.PushReporter) { now := time.Now() handler := kubernetes.ResourceEventHandlerFuncs{ AddFunc: func(obj kubernetes.Resource) { - reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event))) + reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig)) }, UpdateFunc: func(obj kubernetes.Resource) { - reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event))) + reporter.Event(generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig)) }, // ignore events that are deleted DeleteFunc: nil, @@ -107,7 +125,7 @@ func (m *MetricSet) Run(reporter mb.PushReporter) { return } -func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr { +func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig) common.MapStr { eventMeta := common.MapStr{ "timestamp": common.MapStr{ "created": kubernetes.Time(eve.Metadata.CreationTimestamp).UTC(), @@ -123,7 +141,12 @@ func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr { if len(eve.Metadata.Labels) != 0 { labels := make(common.MapStr, len(eve.Metadata.Labels)) for k, v := range eve.Metadata.Labels { - safemapstr.Put(labels, k, v) + if dedotConfig.LabelsDedot { + label := common.DeDot(k) + labels.Put(label, v) + } else { + safemapstr.Put(labels, k, v) + } } eventMeta["labels"] = labels @@ -132,7 +155,12 @@ func generateMapStrFromEvent(eve *kubernetes.Event) common.MapStr { if len(eve.Metadata.Annotations) != 0 { annotations := make(common.MapStr, len(eve.Metadata.Annotations)) for k, v := range eve.Metadata.Annotations { - safemapstr.Put(annotations, k, v) + if dedotConfig.AnnotationsDedot { + annotation := common.DeDot(k) + annotations.Put(annotation, v) + } else { + safemapstr.Put(annotations, k, v) + } } eventMeta["annotations"] = annotations diff --git a/metricbeat/module/kubernetes/event/event_test.go b/metricbeat/module/kubernetes/event/event_test.go new file mode 100644 index 00000000000..5fe869e3694 --- /dev/null +++ b/metricbeat/module/kubernetes/event/event_test.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package event + +import ( + "testing" + + "github.com/ericchiang/k8s/apis/core/v1" + k8s_io_apimachinery_pkg_apis_meta_v1 "github.com/ericchiang/k8s/apis/meta/v1" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestGenerateMapStrFromEvent(t *testing.T) { + labels := map[string]string{ + "app.kubernetes.io/name": "mysql", + "app.kubernetes.io/version": "5.7.21", + "app.kubernetes.io/component": "database", + } + + annotations := map[string]string{ + "prometheus.io/path": "/metrics", + "prometheus.io/port": "9102", + "prometheus.io/scheme": "http", + "prometheus.io/scrape": "false", + } + + expectedLabelsMapStrWithDot := common.MapStr{ + "app": common.MapStr{ + "kubernetes": common.MapStr{ + "io/version": "5.7.21", + "io/component": "database", + "io/name": "mysql", + }, + }, + } + + expectedLabelsMapStrWithDeDot := common.MapStr{ + "app_kubernetes_io/name": "mysql", + "app_kubernetes_io/version": "5.7.21", + "app_kubernetes_io/component": "database", + } + + expectedAnnotationsMapStrWithDot := common.MapStr{ + "prometheus": common.MapStr{ + "io/path": "/metrics", + "io/port": "9102", + "io/scheme": "http", + "io/scrape": "false", + }, + } + + expectedAnnotationsMapStrWithDeDot := common.MapStr{ + "prometheus_io/path": "/metrics", + "prometheus_io/port": "9102", + "prometheus_io/scheme": "http", + "prometheus_io/scrape": "false", + } + + testCases := map[string]struct { + mockEvent v1.Event + expectedMetadata common.MapStr + dedotConfig dedotConfig + }{ + "no dedots": { + mockEvent: v1.Event{ + Metadata: &k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + expectedMetadata: common.MapStr{ + "labels": expectedLabelsMapStrWithDot, + "annotations": expectedAnnotationsMapStrWithDot, + }, + dedotConfig: dedotConfig{ + LabelsDedot: false, + AnnotationsDedot: false, + }, + }, + "dedot labels": { + mockEvent: v1.Event{ + Metadata: &k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + expectedMetadata: common.MapStr{ + "labels": expectedLabelsMapStrWithDeDot, + "annotations": expectedAnnotationsMapStrWithDot, + }, + dedotConfig: dedotConfig{ + LabelsDedot: true, + AnnotationsDedot: false, + }, + }, + "dedot annotatoins": { + mockEvent: v1.Event{ + Metadata: &k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + expectedMetadata: common.MapStr{ + "labels": expectedLabelsMapStrWithDot, + "annotations": expectedAnnotationsMapStrWithDeDot, + }, + dedotConfig: dedotConfig{ + LabelsDedot: false, + AnnotationsDedot: true, + }, + }, + "dedot both labels and annotations": { + mockEvent: v1.Event{ + Metadata: &k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + expectedMetadata: common.MapStr{ + "labels": expectedLabelsMapStrWithDeDot, + "annotations": expectedAnnotationsMapStrWithDeDot, + }, + dedotConfig: dedotConfig{ + LabelsDedot: true, + AnnotationsDedot: true, + }, + }, + } + + for name, test := range testCases { + t.Run(name, func(t *testing.T) { + mapStrOutput := generateMapStrFromEvent(&test.mockEvent, test.dedotConfig) + assert.Equal(t, test.expectedMetadata["labels"], mapStrOutput["metadata"].(common.MapStr)["labels"]) + assert.Equal(t, test.expectedMetadata["annotations"], mapStrOutput["metadata"].(common.MapStr)["annotations"]) + }) + } +} diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 94cb3a9960e..38b08c8e05d 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -20,6 +20,8 @@ # Enriching parameters: #add_metadata: true #in_cluster: true + #labels.dedot: false + #annotations.dedot: false # When used outside the cluster: #in_cluster: false #host: node_name