From c3733f879404a3d5c59429b162feb37e058ae338 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 22 Sep 2020 16:55:49 +0200 Subject: [PATCH] Add cloudfoundry tags to events (#21177) (cherry picked from commit c9ea2cf5b00cd60ba431383c8852e51c4996feae) --- CHANGELOG.next.asciidoc | 1 + x-pack/libbeat/common/cloudfoundry/events.go | 9 ++++ .../common/cloudfoundry/events_test.go | 18 ++++++++ .../_meta/fields.yml | 6 +++ .../cloudfoundry/container/_meta/data.json | 45 +++++++++---------- .../container/container_integration_test.go | 1 + .../cloudfoundry/counter/_meta/data.json | 38 ++++++++-------- .../counter/counter_integration_test.go | 1 + .../module/cloudfoundry/mtest/config.go | 6 +++ .../module/cloudfoundry/value/_meta/data.json | 40 ++++++++--------- .../value/value_integration_test.go | 1 + 11 files changed, 101 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d6c20f254a..5c2416f5ba3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -504,6 +504,7 @@ field. You can revert this change by configuring tags for the module and omittin - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820] - Added experimental dataset `juniper/netscreen`. {pull}20820[20820] - Added experimental dataset `sophos/utm`. {pull}20820[20820] +- Add Cloud Foundry tags in related events. {pull}21177[21177] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index f134585ac65..54850dc5949 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -497,10 +497,19 @@ func baseMap(evt Event) common.MapStr { "timestamp": evt.Timestamp(), }, "envelope": envelopMap(evt), + "tags": dedotedTags(evt.Tags()), }, } } +func dedotedTags(tags map[string]string) common.MapStr { + result := common.MapStr{} + for name, value := range tags { + result[common.DeDot(name)] = value + } + return result +} + func baseMapWithApp(evt EventWithAppID) common.MapStr { base := baseMap(evt) appID := evt.AppGuid() diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index e4fe6f39caf..6dd86ff6ed7 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -89,6 +89,9 @@ func TestEventTypeHttpAccess(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "http": common.MapStr{ "response": common.MapStr{ @@ -164,6 +167,9 @@ func TestEventTypeLog(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "log message", "stream": "stdout", @@ -212,6 +218,9 @@ func TestEventCounter(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -258,6 +267,9 @@ func TestEventValueMetric(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -322,6 +334,9 @@ func TestEventContainerMetric(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -366,6 +381,9 @@ func TestEventError(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "message", "code": int32(100), diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml index bd6e158bb36..1cf512840ca 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml @@ -37,3 +37,9 @@ type: keyword description: > Cloud Foundry organization name + + - name: tags.* + type: object + object_type: keyword + description: > + Cloud Foundry tags. diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 3d1f51694e4..96bb29f1685 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -1,35 +1,32 @@ { - "@timestamp": "2020-02-27T18:09:35.583Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { + "app": { + "id": "ba41cfb8-8658-47d3-a021-2d1a6321aa42" + }, "container": { + "cpu.pct": 0.22287691157883535, + "disk.bytes": 224665600, + "disk.quota.bytes": 268435456, "instance_index": 0, - "cpu.pct": 0.18815706013687805, - "memory.bytes": 23320985, - "memory.quota.bytes": 134217728, - "disk.bytes": 186187776, - "timestamp": "2020-02-27T18:09:35.583Z", - "type": "container", - "disk.quota.bytes": 2122136037 + "memory.bytes": 175937174, + "memory.quota.bytes": 536870912, + "timestamp": "2020-09-22T17:17:47.928768995+02:00" }, "envelope": { - "origin": "rep", - "deployment": "cf", - "ip": "10.144.0.13", - "job": "diego-cell", - "index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "cd6c6b2c-797b-49a9-b1a2-a3b52da80965", + "ip": "192.168.16.32", + "job": "diego_cell", + "origin": "rep" }, - "app": { - "id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a" - } - }, - "event": { - "dataset": "cloudfoundry.container", - "module": "cloudfoundry" - }, - "metricset": { - "name": "container" + "tags": { + "product": "Pivotal Application Service", + "source_id": "ba41cfb8-8658-47d3-a021-2d1a6321aa42" + }, + "type": "container" }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go index e871a5823fc..605f7145f64 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json index d6f155213af..3a38d5caf0c 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -1,29 +1,27 @@ { - "@timestamp": "2020-02-24T18:49:02.674Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "counter": { - "total": 0, - "name": "promhttp_metric_handler_errors_total", - "delta": 0, - "timestamp": "2020-02-24T18:49:02.674Z", - "type": "counter" + "delta": 594, + "name": "egress", + "timestamp": "2020-09-22T17:17:41.065425923+02:00", + "total": 1563170851 }, "envelope": { - "origin": "loggregator_forwarder_agent", - "deployment": "cf", - "ip": "10.144.0.13", - "job": "diego-cell", - "index": "1e58e943-a498-4339-9d5f-2885bf223db9" - } - }, - "event": { - "dataset": "cloudfoundry.counter", - "module": "cloudfoundry" - }, - "metricset": { - "name": "counter" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "15e472dc-d37c-4e88-a4ff-96bd1ee80e57", + "ip": "192.168.16.57", + "job": "doppler", + "origin": "loggregator.doppler" + }, + "tags": { + "metric_version": "2.0", + "product": "Pivotal Application Service", + "source_id": "doppler" + }, + "type": "counter" }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go index 44cb4935e70..f7c13b1a603 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go index c0756c57cca..8f98d528f58 100644 --- a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go @@ -7,6 +7,7 @@ package mtest import ( "testing" + "github.com/elastic/beats/v7/libbeat/beat" cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" ) @@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} { return config } + +// CleanFields removes fields that can contain data of real deployments +func CleanFields(e beat.Event) { + e.Fields.Delete("cloudfoundry.tags.system_domain") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json index bf70762fd38..1413f96463e 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -1,29 +1,27 @@ { - "@timestamp": "2020-02-27T18:20:48.379Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { - "value": { - "unit": "", - "value": 5.125735912782789e-05, - "timestamp": "2020-02-27T18:20:48.379Z", - "type": "value", - "name": "go_memstats_gc_cpu_fraction" - }, "envelope": { - "origin": "loggregator_forwarder_agent", - "deployment": "cf", - "ip": "10.144.0.11", - "job": "scheduler", - "index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d", + "ip": "192.168.16.31", + "job": "diego_cell", + "origin": "rep" + }, + "tags": { + "instance_id": "8895cf68-2e4f-4c50-9842-e73febd65c2d", + "product": "Pivotal Application Service", + "source_id": "rep" + }, + "type": "value", + "value": { + "name": "memoryStats.numBytesAllocatedStack", + "timestamp": "2020-09-22T17:17:40.257407125+02:00", + "unit": "Bytes", + "value": 3899392 } }, - "event": { - "dataset": "cloudfoundry.value", - "module": "cloudfoundry" - }, - "metricset": { - "name": "value" - }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go index 610a0a8e029..af815a3d4b7 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") }