diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 337a97aeb66..ba85dd83a38 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -505,6 +505,7 @@ field. You can revert this change by configuring tags for the module and omittin - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820] - Added experimental dataset `juniper/netscreen`. {pull}20820[20820] - Added experimental dataset `sophos/utm`. {pull}20820[20820] +- Add Cloud Foundry tags in related events. {pull}21177[21177] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index 3006c472ccd..3a0f1756b6e 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -494,10 +494,19 @@ func baseMap(evt Event) common.MapStr { "cloudfoundry": common.MapStr{ "type": evt.String(), "envelope": envelopMap(evt), + "tags": dedotedTags(evt.Tags()), }, } } +func dedotedTags(tags map[string]string) common.MapStr { + result := common.MapStr{} + for name, value := range tags { + result[common.DeDot(name)] = value + } + return result +} + func baseMapWithApp(evt EventWithAppID) common.MapStr { base := baseMap(evt) appID := evt.AppGuid() diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index 91145589969..7dfd9bdcbd7 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -86,6 +86,9 @@ func TestEventTypeHttpAccess(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "http": common.MapStr{ "response": common.MapStr{ @@ -160,6 +163,9 @@ func TestEventTypeLog(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "log message", "stream": "stdout", @@ -207,6 +213,9 @@ func TestEventCounter(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -252,6 +261,9 @@ func TestEventValueMetric(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -315,6 +327,9 @@ func TestEventContainerMetric(t *testing.T) { "app": common.MapStr{ "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", }, + "tags": common.MapStr{ + "tag": "value", + }, }, }, evt.ToFields()) } @@ -358,6 +373,9 @@ func TestEventError(t *testing.T) { "job": "job", "index": "index", }, + "tags": common.MapStr{ + "tag": "value", + }, }, "message": "message", "code": int32(100), diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml index 0ac4a813d50..320cdca5b58 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml @@ -38,6 +38,12 @@ description: > Cloud Foundry organization name. + - name: tags.* + type: object + object_type: keyword + description: > + Cloud Foundry tags. + - name: envelope type: group fields: @@ -65,3 +71,4 @@ type: keyword description: > Name of the deployment that created this event. +>>>>>>> origin/7.x diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 389b0bc54f9..af037200bb4 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -2,23 +2,27 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "app": { - "id": "c0874969-2811-4d80-837a-301fd485a7bf" + "id": "e6bb1d20-64e1-42ff-abda-b502eb8cb2dc" }, "container": { - "cpu.pct": 0.45456369228159, - "disk.bytes": 21970944, + "cpu.pct": 0.2808890817947714, + "disk.bytes": 10100736, "disk.quota.bytes": 1073741824, - "instance_index": 1, - "memory.bytes": 14101162, - "memory.quota.bytes": 67108864 + "instance_index": 3, + "memory.bytes": 14486732, + "memory.quota.bytes": 134217728 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d", - "ip": "192.168.16.31", + "index": "cd6c6b2c-797b-49a9-b1a2-a3b52da80965", + "ip": "192.168.16.32", "job": "diego_cell", "origin": "rep" }, + "tags": { + "product": "Pivotal Application Service", + "source_id": "e6bb1d20-64e1-42ff-abda-b502eb8cb2dc" + }, "type": "container" }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go index e871a5823fc..605f7145f64 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json index b9104346eac..9fd84b94cfa 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -2,17 +2,22 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "counter": { - "delta": 0, - "name": "sinks.dropped", - "total": 89547093 + "delta": 201, + "name": "ingress", + "total": 512406486 }, "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "9adedbd0-3af3-47af-9707-99016050b123", - "ip": "192.168.16.54", + "index": "d0d52df7-be5e-44ec-b643-988dd9a85ca8", + "ip": "192.168.16.53", "job": "doppler", "origin": "loggregator.doppler" }, + "tags": { + "metric_version": "2.0", + "product": "Pivotal Application Service", + "source_id": "doppler" + }, "type": "counter" }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go index 44cb4935e70..f7c13b1a603 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") } diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go index c0756c57cca..8f98d528f58 100644 --- a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go @@ -7,6 +7,7 @@ package mtest import ( "testing" + "github.com/elastic/beats/v7/libbeat/beat" cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" ) @@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} { return config } + +// CleanFields removes fields that can contain data of real deployments +func CleanFields(e beat.Event) { + e.Fields.Delete("cloudfoundry.tags.system_domain") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json index feeb2ce72ba..6b663daec72 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -3,16 +3,19 @@ "cloudfoundry": { "envelope": { "deployment": "cf-6b7aee31c8d07637ad78", - "index": "439bb519-3e68-4d08-a973-8f013f364d61", - "ip": "192.168.16.18", - "job": "cloud_controller", - "origin": "routing_api" + "index": "6f2f5d74-9375-4d82-870f-c09e6a74cf3a", + "ip": "192.168.16.46", + "job": "diego_cell", + "origin": "netmon" + }, + "tags": { + "source_id": "netmon" }, "type": "value", "value": { "name": "memoryStats.lastGCPauseTimeNS", "unit": "count", - "value": 77499 + "value": 366021 } }, "service": { diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go index 610a0a8e029..af815a3d4b7 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go @@ -49,5 +49,6 @@ func TestData(t *testing.T) { require.NotEmpty(t, events) beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mtest.CleanFields(beatEvent) mbtest.WriteEventToDataJSON(t, beatEvent, "") }