Skip to content

Commit

Permalink
Some fixes in Cloud Foundry fields (elastic#21175)
Browse files Browse the repository at this point in the history
Remove redundant and undocumented `cloudfoundry.*.timestamp` fields.
Add missing documentation about envelope fields.
Update `data.json` files in metricbeat module.
  • Loading branch information
jsoriano authored Sep 21, 2020
1 parent 43354ff commit 3527264
Show file tree
Hide file tree
Showing 7 changed files with 84 additions and 90 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608]
- Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
- Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]

*Auditbeat*

Expand Down
5 changes: 1 addition & 4 deletions x-pack/libbeat/common/cloudfoundry/events.go
Original file line number Diff line number Diff line change
Expand Up @@ -492,10 +492,7 @@ func envelopMap(evt Event) common.MapStr {
func baseMap(evt Event) common.MapStr {
return common.MapStr{
"cloudfoundry": common.MapStr{
"type": evt.String(),
evt.String(): common.MapStr{
"timestamp": evt.Timestamp(),
},
"type": evt.String(),
"envelope": envelopMap(evt),
},
}
Expand Down
22 changes: 7 additions & 15 deletions x-pack/libbeat/common/cloudfoundry/events_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@ func TestEventTypeHttpAccess(t *testing.T) {
assert.Equal(t, common.MapStr{
"cloudfoundry": common.MapStr{
"type": "access",
"access": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
},
"envelope": common.MapStr{
"origin": "origin",
"deployment": "deployment",
Expand Down Expand Up @@ -148,7 +145,6 @@ func TestEventTypeLog(t *testing.T) {
"cloudfoundry": common.MapStr{
"type": "log",
"log": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
"source": common.MapStr{
"instance": evt.SourceID(),
"type": evt.SourceType(),
Expand Down Expand Up @@ -200,10 +196,9 @@ func TestEventCounter(t *testing.T) {
"cloudfoundry": common.MapStr{
"type": "counter",
"counter": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
"name": "name",
"delta": uint64(10),
"total": uint64(999),
"name": "name",
"delta": uint64(10),
"total": uint64(999),
},
"envelope": common.MapStr{
"origin": "origin",
Expand Down Expand Up @@ -246,10 +241,9 @@ func TestEventValueMetric(t *testing.T) {
"cloudfoundry": common.MapStr{
"type": "value",
"value": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
"name": "name",
"value": 10.1,
"unit": "unit",
"name": "name",
"value": 10.1,
"unit": "unit",
},
"envelope": common.MapStr{
"origin": "origin",
Expand Down Expand Up @@ -304,7 +298,6 @@ func TestEventContainerMetric(t *testing.T) {
"cloudfoundry": common.MapStr{
"type": "container",
"container": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
"instance_index": int32(1),
"cpu.pct": 0.2,
"memory.bytes": uint64(1024),
Expand Down Expand Up @@ -356,8 +349,7 @@ func TestEventError(t *testing.T) {
"cloudfoundry": common.MapStr{
"type": "error",
"error": common.MapStr{
"timestamp": time.Unix(0, 1587469726082),
"source": "source",
"source": "source",
},
"envelope": common.MapStr{
"origin": "origin",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,57 @@
- name: app.id
type: keyword
description: >
Cloud Foundry application ID
Cloud Foundry application identifier.
- name: app.name
type: keyword
description: >
Cloud Foundry application name
Cloud Foundry application name.
- name: space.id
type: keyword
description: >
Cloud Foundry space name
Cloud Foundry space identifier.
- name: space.name
type: keyword
description: >
Cloud Foundry space name
Cloud Foundry space name.
- name: org.id
type: keyword
description: >
Cloud Foundry organization ID
Cloud Foundry organization identifier.
- name: org.name
type: keyword
description: >
Cloud Foundry organization name
Cloud Foundry organization name.
- name: envelope
type: group
fields:
- name: deployment
type: keyword
description: >
Identifier of the Cloud Foundry deployment where this event was created.
- name: index
type: keyword
description: >
Envelope index.
- name: ip
type: ip
description: >
IP address of the node where the job that created this event is running.
- name: job
type: keyword
description: >
Job name.
- name: origin
type: keyword
description: >
Name of the deployment that created this event.
42 changes: 17 additions & 25 deletions x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,35 +1,27 @@
{
"@timestamp": "2020-02-27T18:09:35.583Z",
"@timestamp": "2017-10-12T08:05:34.853Z",
"cloudfoundry": {
"app": {
"id": "c0874969-2811-4d80-837a-301fd485a7bf"
},
"container": {
"instance_index": 0,
"cpu.pct": 0.18815706013687805,
"memory.bytes": 23320985,
"memory.quota.bytes": 134217728,
"disk.bytes": 186187776,
"timestamp": "2020-02-27T18:09:35.583Z",
"type": "container",
"disk.quota.bytes": 2122136037
"cpu.pct": 0.45456369228159,
"disk.bytes": 21970944,
"disk.quota.bytes": 1073741824,
"instance_index": 1,
"memory.bytes": 14101162,
"memory.quota.bytes": 67108864
},
"envelope": {
"origin": "rep",
"deployment": "cf",
"ip": "10.144.0.13",
"job": "diego-cell",
"index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0"
"deployment": "cf-6b7aee31c8d07637ad78",
"index": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
"ip": "192.168.16.31",
"job": "diego_cell",
"origin": "rep"
},
"app": {
"id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a"
}
},
"event": {
"dataset": "cloudfoundry.container",
"module": "cloudfoundry"
},
"metricset": {
"name": "container"
"type": "container"
},
"service": {
"type": "cloudfoundry"
}
}
}
30 changes: 11 additions & 19 deletions x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,29 +1,21 @@
{
"@timestamp": "2020-02-24T18:49:02.674Z",
"@timestamp": "2017-10-12T08:05:34.853Z",
"cloudfoundry": {
"counter": {
"total": 0,
"name": "promhttp_metric_handler_errors_total",
"delta": 0,
"timestamp": "2020-02-24T18:49:02.674Z",
"type": "counter"
"name": "sinks.dropped",
"total": 89547093
},
"envelope": {
"origin": "loggregator_forwarder_agent",
"deployment": "cf",
"ip": "10.144.0.13",
"job": "diego-cell",
"index": "1e58e943-a498-4339-9d5f-2885bf223db9"
}
},
"event": {
"dataset": "cloudfoundry.counter",
"module": "cloudfoundry"
},
"metricset": {
"name": "counter"
"deployment": "cf-6b7aee31c8d07637ad78",
"index": "9adedbd0-3af3-47af-9707-99016050b123",
"ip": "192.168.16.54",
"job": "doppler",
"origin": "loggregator.doppler"
},
"type": "counter"
},
"service": {
"type": "cloudfoundry"
}
}
}
34 changes: 13 additions & 21 deletions x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,29 +1,21 @@
{
"@timestamp": "2020-02-27T18:20:48.379Z",
"@timestamp": "2017-10-12T08:05:34.853Z",
"cloudfoundry": {
"value": {
"unit": "",
"value": 5.125735912782789e-05,
"timestamp": "2020-02-27T18:20:48.379Z",
"type": "value",
"name": "go_memstats_gc_cpu_fraction"
},
"envelope": {
"origin": "loggregator_forwarder_agent",
"deployment": "cf",
"ip": "10.144.0.11",
"job": "scheduler",
"index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f"
"deployment": "cf-6b7aee31c8d07637ad78",
"index": "439bb519-3e68-4d08-a973-8f013f364d61",
"ip": "192.168.16.18",
"job": "cloud_controller",
"origin": "routing_api"
},
"type": "value",
"value": {
"name": "memoryStats.lastGCPauseTimeNS",
"unit": "count",
"value": 77499
}
},
"event": {
"dataset": "cloudfoundry.value",
"module": "cloudfoundry"
},
"metricset": {
"name": "value"
},
"service": {
"type": "cloudfoundry"
}
}
}

0 comments on commit 3527264

Please sign in to comment.