diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index 3a0f1756b6e..adaa944773c 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -490,21 +490,45 @@ func envelopMap(evt Event) common.MapStr { } func baseMap(evt Event) common.MapStr { - return common.MapStr{ - "cloudfoundry": common.MapStr{ - "type": evt.String(), - "envelope": envelopMap(evt), - "tags": dedotedTags(evt.Tags()), - }, + tags, meta := tagsToMeta(evt.Tags()) + cf := common.MapStr{ + "type": evt.String(), + "envelope": envelopMap(evt), + } + if len(tags) > 0 { + cf["tags"] = tags + } + result := common.MapStr{ + "cloudfoundry": cf, } + if len(meta) > 0 { + result.DeepUpdate(meta) + } + return result } -func dedotedTags(tags map[string]string) common.MapStr { - result := common.MapStr{} - for name, value := range tags { - result[common.DeDot(name)] = value +func tagsToMeta(eventTags map[string]string) (tags common.MapStr, meta common.MapStr) { + tags = common.MapStr{} + meta = common.MapStr{} + for name, value := range eventTags { + switch name { + case "app_id": + meta.Put("cloudfoundry.app.id", value) + case "app_name": + meta.Put("cloudfoundry.app.name", value) + case "space_id": + meta.Put("cloudfoundry.space.id", value) + case "space_name": + meta.Put("cloudfoundry.space.name", value) + case "organization_id": + meta.Put("cloudfoundry.org.id", value) + case "organization_name": + meta.Put("cloudfoundry.org.name", value) + default: + tags[common.DeDot(name)] = value + } } - return result + return tags, meta } func baseMapWithApp(evt EventWithAppID) common.MapStr { diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index 7dfd9bdcbd7..af924b33ba3 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -382,6 +382,87 @@ func TestEventError(t *testing.T) { }, evt.ToFields()) } +func TestEventTagsWithMetadata(t *testing.T) { + eventType := events.Envelope_LogMessage + message := "log message" + messageType := events.LogMessage_OUT + timestamp := int64(1587469726082) + appID := "f47ac10b-58cc-4372-a567-0e02b2c3d479" + sourceType := "source_type" + sourceInstance := "source_instance" + cfEvt := makeEnvelope(&eventType) + tags := map[string]string{ + "app_id": appID, + "app_name": "some-app", + "space_id": "e1114e92-155c-11eb-ada9-27b81025a657", + "space_name": "some-space", + "organization_id": "baeef1ba-155c-11eb-a1af-8f14964c35d2", + "organization_name": "some-org", + "custom_tag": "foo", + } + cfEvt.Tags = tags + cfEvt.LogMessage = &events.LogMessage{ + Message: []byte(message), + MessageType: &messageType, + Timestamp: ×tamp, + AppId: &appID, + SourceType: &sourceType, + SourceInstance: &sourceInstance, + } + evt := newEventLog(cfEvt) + + assert.Equal(t, EventTypeLog, evt.EventType()) + assert.Equal(t, "log", evt.String()) + assert.Equal(t, "origin", evt.Origin()) + assert.Equal(t, time.Unix(0, 1587469726082), evt.Timestamp()) + assert.Equal(t, "deployment", evt.Deployment()) + assert.Equal(t, "job", evt.Job()) + assert.Equal(t, "index", evt.Index()) + assert.Equal(t, "ip", evt.IP()) + assert.Equal(t, tags, evt.Tags()) + assert.Equal(t, "f47ac10b-58cc-4372-a567-0e02b2c3d479", evt.AppGuid()) + assert.Equal(t, "log message", evt.Message()) + assert.Equal(t, EventLogMessageTypeStdout, evt.MessageType()) + assert.Equal(t, "source_type", evt.SourceType()) + assert.Equal(t, "source_instance", evt.SourceID()) + + assert.Equal(t, common.MapStr{ + "cloudfoundry": common.MapStr{ + "type": "log", + "log": common.MapStr{ + "source": common.MapStr{ + "instance": evt.SourceID(), + "type": evt.SourceType(), + }, + }, + "envelope": common.MapStr{ + "origin": "origin", + "deployment": "deployment", + "ip": "ip", + "job": "job", + "index": "index", + }, + "app": common.MapStr{ + "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "name": "some-app", + }, + "space": common.MapStr{ + "id": "e1114e92-155c-11eb-ada9-27b81025a657", + "name": "some-space", + }, + "org": common.MapStr{ + "id": "baeef1ba-155c-11eb-a1af-8f14964c35d2", + "name": "some-org", + }, + "tags": common.MapStr{ + "custom_tag": "foo", + }, + }, + "message": "log message", + "stream": "stdout", + }, evt.ToFields()) +} + func makeEnvelope(eventType *events.Envelope_EventType) *events.Envelope { timestamp := int64(1587469726082) origin := "origin" diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go index a6b8bd16566..50f551af969 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go @@ -69,6 +69,10 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) { // wrong type or not set return event, nil } + if hasMetadataFields(event) { + // nothing to do, fields already present + return event, nil + } app, err := d.client.GetAppByGuid(val) if err != nil { d.log.Debugf("failed to get application info for GUID(%s): %v", val, err) @@ -108,3 +112,21 @@ func (d *addCloudFoundryMetadata) Close() error { } return nil } + +var metadataFields = []string{ + "cloudfoundry.app.id", + "cloudfoundry.app.name", + "cloudfoundry.space.id", + "cloudfoundry.space.name", + "cloudfoundry.org.id", + "cloudfoundry.org.name", +} + +func hasMetadataFields(event *beat.Event) bool { + for _, name := range metadataFields { + if value, err := event.GetValue(name); value == "" || err != nil { + return false + } + } + return true +} diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go index 95a7073321e..34fe866104d 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go @@ -82,6 +82,62 @@ func TestCFAppNotFound(t *testing.T) { assert.Equal(t, evt, *observed) } +func TestCFAppMetadataAlreadyPresent(t *testing.T) { + guid := mustCreateFakeGuid() + app := cloudfoundry.AppMeta{ + Guid: guid, + Name: "My Fake App", + SpaceGuid: mustCreateFakeGuid(), + SpaceName: "My Fake Space", + OrgGuid: mustCreateFakeGuid(), + OrgName: "My Fake Org", + } + p := addCloudFoundryMetadata{ + log: logp.NewLogger("add_cloudfoundry_metadata"), + client: &fakeClient{app}, + } + + evt := beat.Event{ + Fields: common.MapStr{ + "cloudfoundry": common.MapStr{ + "app": common.MapStr{ + "id": guid, + "name": "Other App Name", + }, + "space": common.MapStr{ + "id": app.SpaceGuid, + "name": app.SpaceName, + }, + "org": common.MapStr{ + "id": app.OrgGuid, + "name": app.OrgName, + }, + }, + }, + } + expected := beat.Event{ + Fields: common.MapStr{ + "cloudfoundry": common.MapStr{ + "app": common.MapStr{ + "id": guid, + "name": "Other App Name", + }, + "space": common.MapStr{ + "id": app.SpaceGuid, + "name": app.SpaceName, + }, + "org": common.MapStr{ + "id": app.OrgGuid, + "name": app.OrgName, + }, + }, + }, + } + observed, err := p.Run(&evt) + assert.NoError(t, err) + assert.Equal(t, expected, *observed) +} + func TestCFAppUpdated(t *testing.T) { guid := mustCreateFakeGuid() app := cloudfoundry.AppMeta{ diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc index 67e89c8173b..e5123ae75a0 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc @@ -20,6 +20,11 @@ Each event is annotated with: * Organization ID * Organization Name +NOTE: Pivotal Application Service and Tanzu Application Service include this +metadata in all events from the firehose since version 2.8. In these cases the +metadata in the events is used, and `add_cloudfoundry_metadata` processor +doesn't modify these fields. + [source,yaml] ------------------------------------------------------------------------------- diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 16a4f1ef128..57d788406ef 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -2,26 +2,39 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "app": { - "id": "3ce55e14-de73-49af-836d-adc93f3fee39" + "id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c", + "name": "log-gen" }, "container": { - "cpu.pct": 0.19431789913648675, - "disk.bytes": 16678912, - "disk.quota.bytes": 33554432, - "instance_index": 0, - "memory.bytes": 8529920, - "memory.quota.bytes": 33554432 + "cpu.pct": 4.231873716293137, + "disk.bytes": 122691584, + "disk.quota.bytes": 1073741824, + "instance_index": 3, + "memory.bytes": 52250065, + "memory.quota.bytes": 1073741824 }, "envelope": { - "deployment": "cf-6b7aee31c8d07637ad78", - "index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668", - "ip": "192.168.16.51", + "deployment": "cf-9c11cd01425665e2ed6d", + "index": "9e1a45be-f8a4-44ef-9c34-22f6e51ce4c7", + "ip": "192.168.16.37", "job": "diego_cell", "origin": "rep" }, + "org": { + "id": "4af89198-dd33-4542-9915-f489542bc058", + "name": "elastic-logging-org" + }, + "space": { + "id": "10ed1559-e399-4034-babf-6424ef888dc1", + "name": "logging-space" + }, "tags": { - "product": "Pivotal Application Service", - "source_id": "3ce55e14-de73-49af-836d-adc93f3fee39" + "instance_id": "3", + "process_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c", + "process_instance_id": "75568c86-b9e0-4330-4784-928b", + "process_type": "web", + "product": "VMware Tanzu Application Service", + "source_id": "8d165a12-fbd8-40cb-b71a-5bc6086df04c" }, "type": "container" },