diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3d28e083b0a..09c6e994d62 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -66,7 +66,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix k8s pods annotations broken schema. {pull}16554[16554] - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} - Fix `NewContainerMetadataEnricher` to use default config for kubernetes module. {pull}16857[16857] -- Improve some logging messages for add_kubernetes_metadata processor {pull}16866{16866} +- Improve some logging messages for add_kubernetes_metadata processor {pull}16866[16866] +- Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834] *Auditbeat* diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go index d81f79f28d5..a2b474281cf 100644 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ b/libbeat/common/kubernetes/metadata/metadata.go @@ -18,6 +18,8 @@ package metadata import ( + "strings" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/libbeat/common/safemapstr" @@ -40,3 +42,11 @@ func WithFields(key string, value interface{}) FieldOptions { safemapstr.Put(meta, key, value) } } + +// WithLabels FieldOption allows adding labels under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithLabels(kind string) FieldOptions { + return func(meta common.MapStr) { + safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } +} diff --git a/libbeat/common/kubernetes/metadata/namespace.go b/libbeat/common/kubernetes/metadata/namespace.go index 92dfbc55b2a..dd3011a6c47 100644 --- a/libbeat/common/kubernetes/metadata/namespace.go +++ b/libbeat/common/kubernetes/metadata/namespace.go @@ -60,17 +60,16 @@ func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) common.M return nil } - obj, ok, _ := n.store.GetByKey(name) - if !ok { - return nil - } + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } - no, ok := obj.(*kubernetes.Namespace) - if !ok { - return nil + return n.Generate(no, opts...) } - return n.Generate(no, opts...) + return nil } func flattenMetadata(in common.MapStr) common.MapStr { @@ -84,7 +83,6 @@ func flattenMetadata(in common.MapStr) common.MapStr { if !ok { return nil } - for k, v := range fields { if k == "name" { out[resource] = v @@ -93,5 +91,15 @@ func flattenMetadata(in common.MapStr) common.MapStr { } } + rawLabels, err := in.GetValue("labels") + if err != nil { + return out + } + labels, ok := rawLabels.(common.MapStr) + if !ok { + return out + } + out[resource+"_labels"] = labels + return out } diff --git a/libbeat/common/kubernetes/metadata/namespace_test.go b/libbeat/common/kubernetes/metadata/namespace_test.go index 5ac460d7ca7..40763887197 100644 --- a/libbeat/common/kubernetes/metadata/namespace_test.go +++ b/libbeat/common/kubernetes/metadata/namespace_test.go @@ -58,17 +58,18 @@ func TestNamespace_Generate(t *testing.T) { }, }, // Use this for 8.0 - /*output: common.MapStr{ - "namespace": common.MapStr{ - "name": "obj", - "uid": uid, - "labels": common.MapStr{ - "foo": "bar", + /* + output: common.MapStr{ + "namespace": common.MapStr{ + "name": name, + "uid": uid, + "labels": common.MapStr{ + "foo": "bar", + }, }, - }, - },*/ + },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", @@ -114,7 +115,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { /* output: common.MapStr{ "namespace": common.MapStr{ - "name": "obj", + "name": name, "uid": uid, "labels": common.MapStr{ "foo": "bar", @@ -122,7 +123,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { }, },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go index 226e84710c2..100db49c6d8 100644 --- a/libbeat/common/kubernetes/metadata/node_test.go +++ b/libbeat/common/kubernetes/metadata/node_test.go @@ -61,9 +61,9 @@ func TestNode_Generate(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, @@ -106,9 +106,9 @@ func TestNode_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index c25db1f7a76..9234f54533b 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,11 +49,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) - // TODO: remove this call when moving to 8.0 - out = p.exportPodLabelsAndAnnotations(out) if p.node != nil { - meta := p.node.GenerateFromName(po.Spec.NodeName) + meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node")) if meta != nil { out.Put("node", meta["node"]) } else { @@ -91,21 +89,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } - -func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr { - labels, err := in.GetValue("pod.labels") - if err != nil { - return in - } - in.Put("labels", labels) - in.Delete("pod.labels") - - annotations, err := in.GetValue("pod.annotations") - if err != nil { - return in - } - in.Put("annotations", annotations) - in.Delete("pod.annotations") - - return in -} diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go index e4f4b8ca028..f8f986fe42c 100644 --- a/libbeat/common/kubernetes/metadata/resource.go +++ b/libbeat/common/kubernetes/metadata/resource.go @@ -98,11 +98,11 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie } if len(labelMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".labels", labelMap) + safemapstr.Put(meta, "labels", labelMap) } if len(annotationsMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".annotations", annotationsMap) + safemapstr.Put(meta, "annotations", annotationsMap) } for _, option := range options { diff --git a/libbeat/common/kubernetes/metadata/resource_test.go b/libbeat/common/kubernetes/metadata/resource_test.go index 6df542742f0..f4c25ecbad5 100644 --- a/libbeat/common/kubernetes/metadata/resource_test.go +++ b/libbeat/common/kubernetes/metadata/resource_test.go @@ -62,9 +62,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -99,9 +99,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go index a574f1a7b7a..0fc06af3a0b 100644 --- a/libbeat/common/kubernetes/metadata/service_test.go +++ b/libbeat/common/kubernetes/metadata/service_test.go @@ -64,9 +64,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -101,9 +101,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -153,9 +153,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -190,9 +190,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -262,10 +262,19 @@ func TestService_GenerateWithNamespace(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, + "labels": common.MapStr{ + "foo": "bar", + }, + // Use this for 8.0 + /* + "namespace": common.MapStr{ + "name": "default", + "uid": uid, + "labels": common.MapStr{ + "nskey": "nsvalue", + }, + },*/ "namespace": "default", "namespace_uid": uid, "namespace_labels": common.MapStr{