Skip to content

Commit

Permalink
Backport elastic#29133 to 7.17
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaelKatsoulis
MichaelKatsoulis committed Sep 22, 2022
1 parent bd956ba commit 4fa2000
Show file tree
Hide file tree
Showing 13 changed files with 323 additions and 103 deletions.
3 changes: 0 additions & 3 deletions libbeat/autodiscover/providers/kubernetes/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,6 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
options.Namespace = config.Namespace
}
metaConf := config.AddResourceMetadata
if metaConf == nil {
metaConf = metadata.GetDefaultResourceMetadataConfig()
}
nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
if err != nil {
logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
Expand Down
2 changes: 1 addition & 1 deletion libbeat/common/kubernetes/metadata/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func GetPodMetaGen(
if namespaceWatcher != nil && metaConf.Namespace.Enabled() {
namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client())
}
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen)
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf)

return metaGen
}
Expand Down
3 changes: 3 additions & 0 deletions libbeat/common/kubernetes/metadata/namespace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ func TestNamespace_Generate(t *testing.T) {
UID: types.UID(uid),
Labels: map[string]string{
"foo": "bar",
"key": "value",
},
Annotations: map[string]string{
"spam": "baz",
"key": "value",
},
},
TypeMeta: metav1.TypeMeta{
Expand Down Expand Up @@ -88,6 +90,7 @@ func TestNamespace_Generate(t *testing.T) {
}

cfg, err := common.NewConfigFrom(Config{
IncludeLabels: []string{"foo"},
IncludeAnnotations: []string{"spam"},
})
if err != nil {
Expand Down
29 changes: 18 additions & 11 deletions libbeat/common/kubernetes/metadata/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@ import (
)

type pod struct {
store cache.Store
client k8s.Interface
node MetaGen
namespace MetaGen
resource *Resource
store cache.Store
client k8s.Interface
node MetaGen
namespace MetaGen
resource *Resource
addResourceMetadata *AddResourceMetadataConfig
}

// NewPodMetadataGenerator creates a metagen for pod resources
Expand All @@ -42,13 +43,19 @@ func NewPodMetadataGenerator(
pods cache.Store,
client k8s.Interface,
node MetaGen,
namespace MetaGen) MetaGen {
namespace MetaGen,
addResourceMetadata *AddResourceMetadataConfig) MetaGen {

if addResourceMetadata == nil {
addResourceMetadata = GetDefaultResourceMetadataConfig()
}
return &pod{
resource: NewResourceMetadataGenerator(cfg, client),
store: pods,
node: node,
namespace: namespace,
client: client,
resource: NewResourceMetadataGenerator(cfg, client),
store: pods,
node: node,
namespace: namespace,
client: client,
addResourceMetadata: addResourceMetadata,
}
}

Expand Down
154 changes: 151 additions & 3 deletions libbeat/common/kubernetes/metadata/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,7 @@ func TestPod_Generate(t *testing.T) {
})
assert.Nil(t, err)

metagen := NewPodMetadataGenerator(config, nil, client, nil, nil)
metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, nil)
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.output, metagen.Generate(test.input))
Expand Down Expand Up @@ -551,7 +551,7 @@ func TestPod_GenerateFromName(t *testing.T) {
assert.Nil(t, err)
pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
pods.Add(test.input)
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil)
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, nil)

accessor, err := meta.Accessor(test.input)
require.Nil(t, err)
Expand Down Expand Up @@ -673,7 +673,155 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
namespaces.Add(test.namespace)
nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client)

metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta)
metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, nil)
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.output, metagen.Generate(test.input))
})
}
}

func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) {
client := k8sfake.NewSimpleClientset()
uid := "005f3b90-4b9d-12f8-acf0-31020a840133"
namespace := "default"
name := "obj"
boolean := true

tests := []struct {
input kubernetes.Resource
node kubernetes.Resource
namespace kubernetes.Resource
output common.MapStr
name string
}{
{
name: "test simple object",
input: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
UID: types.UID(uid),
Namespace: namespace,
Labels: map[string]string{
"app.kubernetes.io/component": "exporter",
},
Annotations: map[string]string{
"app": "production",
},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "apps",
Kind: "ReplicaSet",
Name: "nginx-rs",
UID: "005f3b90-4b9d-12f8-acf0-31020a8409087",
Controller: &boolean,
},
},
},
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},

Spec: v1.PodSpec{
NodeName: "testnode",
},
Status: v1.PodStatus{PodIP: "127.0.0.5"},
},
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "testnode",
UID: types.UID(uid),
Labels: map[string]string{
"nodekey": "nodevalue",
"nodekey2": "nodevalue2",
},
Annotations: map[string]string{},
},
TypeMeta: metav1.TypeMeta{
Kind: "Node",
APIVersion: "v1",
},
Status: v1.NodeStatus{
Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
},
},
namespace: &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
UID: types.UID(uid),
Labels: map[string]string{
"app.kubernetes.io/name": "kube-state-metrics",
"nskey2": "nsvalue2",
},
Annotations: map[string]string{},
},
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: "v1",
},
},
output: common.MapStr{"kubernetes": common.MapStr{
"pod": common.MapStr{
"name": "obj",
"uid": uid,
"ip": "127.0.0.5",
},
"namespace": "default",
"namespace_uid": uid,
"namespace_labels": common.MapStr{
"app_kubernetes_io/name": "kube-state-metrics",
},
"node": common.MapStr{
"name": "testnode",
"uid": uid,
"labels": common.MapStr{
"nodekey2": "nodevalue2",
},
"hostname": "node1",
},
"labels": common.MapStr{
"app_kubernetes_io/component": "exporter",
},
"annotations": common.MapStr{
"app": "production",
},
"replicaset": common.MapStr{
"name": "nginx-rs",
},
}},
},
}

for _, test := range tests {
config, err := common.NewConfigFrom(map[string]interface{}{
"include_annotations": []string{"app"},
})

assert.NoError(t, err)

namespaceConfig, _ := common.NewConfigFrom(map[string]interface{}{
"include_labels": []string{"app.kubernetes.io/name"},
})
nodeConfig, _ := common.NewConfigFrom(map[string]interface{}{
"include_labels": []string{"nodekey2"},
})
metaConfig := AddResourceMetadataConfig{
Namespace: namespaceConfig,
Node: nodeConfig,
}

pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
pods.Add(test.input)

nodes := cache.NewStore(cache.MetaNamespaceKeyFunc)
nodes.Add(test.node)
nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client)

namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc)
namespaces.Add(test.namespace)
nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client)

metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, &metaConfig)
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.output, metagen.Generate(test.input))
})
Expand Down
2 changes: 1 addition & 1 deletion libbeat/common/kubernetes/metadata/resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ...
return nil
}

labelMap := common.MapStr{}
var labelMap common.MapStr
if len(r.config.IncludeLabels) == 0 {
labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot)
} else {
Expand Down
8 changes: 4 additions & 4 deletions libbeat/processors/add_kubernetes_metadata/indexers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
"github.com/elastic/beats/v7/libbeat/common/kubernetes"
)

var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil)
var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, nil)

func TestPodIndexer(t *testing.T) {
var testConfig = common.NewConfig()
Expand Down Expand Up @@ -90,7 +90,7 @@ func TestPodIndexer(t *testing.T) {
func TestPodUIDIndexer(t *testing.T) {
var testConfig = common.NewConfig()

metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil)
metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, nil)

podUIDIndexer, err := NewPodUIDIndexer(*testConfig, metaGenWithPodUID)
assert.Nil(t, err)
Expand Down Expand Up @@ -301,7 +301,7 @@ func TestFilteredGenMeta(t *testing.T) {
})
assert.Nil(t, err)

filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil)
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, nil)

podIndexer, err = NewPodNameIndexer(*testConfig, filteredGen)
assert.Nil(t, err)
Expand Down Expand Up @@ -338,7 +338,7 @@ func TestFilteredGenMetaExclusion(t *testing.T) {
})
assert.Nil(t, err)

filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil)
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, nil)

podIndexer, err := NewPodNameIndexer(*testConfig, filteredGen)
assert.Nil(t, err)
Expand Down
3 changes: 0 additions & 3 deletions libbeat/processors/add_kubernetes_metadata/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,6 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
}

metaConf := config.AddResourceMetadata
if metaConf == nil {
metaConf = metadata.GetDefaultResourceMetadataConfig()
}

options := kubernetes.WatchOptions{
SyncTimeout: config.SyncPeriod,
Expand Down
9 changes: 9 additions & 0 deletions metricbeat/module/kubernetes/_meta/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
# To configure additionally node and namespace metadata, added to pod, service and container resource types,
# `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
# namespace:
# include_labels: ["namespacelabel1"]
# node:
# include_labels: ["nodelabel2"]
# include_annotations: ["nodeannotation1"]

# State metrics from kube-state-metrics service:
#- module: kubernetes
Expand Down
Loading

0 comments on commit 4fa2000

Please sign in to comment.