Introduce log.source.address and log.file.path for 7.x compatiblity

Related to #8902 but adding the fields instead of replacing
elastic · Dec 12, 2018 · 8750ba1 · 8750ba1
1 parent 425c3ed
commit 8750ba1
Show file tree

Hide file tree

Showing 43 changed files with 293 additions and 8 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -99,6 +99,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 - Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768].
 - Allow to force CRI format parsing for better performance {pull}8424[8424]
 - Add event.dataset to module events. {pull}9457[9457]
+- Add field log.source.address and log.file.path to replace source. {pull}9435[9435]
 
 *Heartbeat*
 
@@ -126,6 +127,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 *Affecting all Beats*
 
 *Filebeat*
+- Deprecate field source. Will be replaced by log.source.address and log.file.path in 7.0. {pull}9435[9435]
 
 *Heartbeat*
 

diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml
@@ -116,6 +116,16 @@
       description: >
         This field contains the flags of the event.
 
+    - name: log.source.address
+      type: keyword
+      description: >
+        Log source address.
+
+    - name: log.file.path
+      type: keyword
+      description: >
+        Log source path.
+
     - name: event.created
       type: date
       description: >

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -3126,6 +3126,26 @@ Logging level.
 This field contains the flags of the event.
 
 
+--
+
+*`log.source.address`*::
++
+--
+type: keyword
+
+Log source address.
+
+
+--
+
+*`log.file.path`*::
++
+--
+type: keyword
+
+Log source path.
+
+
 --
 
 *`event.created`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/input/log/harvester.go b/filebeat/input/log/harvester.go
@@ -310,6 +310,11 @@ func (h *Harvester) Run() error {
 			fields := common.MapStr{
 				"source": state.Source,
 				"offset": startingOffset, // Offset here is the offset before the starting char.
+				"log": common.MapStr{
+					"file": common.MapStr{
+						"path": state.Source,
+					},
+				},
 			}
 			fields.DeepUpdate(message.Fields)
 

diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go
@@ -204,6 +204,11 @@ func createEvent(ev *event, metadata inputsource.NetworkMetadata, timezone *time
 	f := common.MapStr{
 		"message": strings.TrimRight(ev.Message(), "\n"),
 		"source":  metadata.RemoteAddr.String(),
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": metadata.RemoteAddr.String(),
+			},
+		},
 	}
 
 	syslog := common.MapStr{}

diff --git a/filebeat/input/syslog/input_test.go b/filebeat/input/syslog/input_test.go
@@ -55,6 +55,11 @@ func TestWhenPriorityIsSet(t *testing.T) {
 			"facility_label": "user-level",
 			"priority":       13,
 		},
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": "127.0.0.1",
+			},
+		},
 	}
 
 	assert.Equal(t, expected, event.Fields)
@@ -77,6 +82,11 @@ func TestWhenPriorityIsNotSet(t *testing.T) {
 		},
 		"event":  common.MapStr{},
 		"syslog": common.MapStr{},
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": "127.0.0.1",
+			},
+		},
 	}
 
 	assert.Equal(t, expected, event.Fields)

diff --git a/filebeat/input/tcp/input.go b/filebeat/input/tcp/input.go
@@ -126,6 +126,11 @@ func createEvent(raw []byte, metadata inputsource.NetworkMetadata) *util.Data {
 		Fields: common.MapStr{
 			"message": string(raw),
 			"source":  metadata.RemoteAddr.String(),
+			"log": common.MapStr{
+				"source": common.MapStr{
+					"address": metadata.RemoteAddr.String(),
+				},
+			},
 		},
 	}
 	return data

diff --git a/filebeat/input/udp/input.go b/filebeat/input/udp/input.go
@@ -75,6 +75,11 @@ func NewInput(
 			Fields: common.MapStr{
 				"message": string(data),
 				"source":  metadata.RemoteAddr.String(),
+				"log": common.MapStr{
+					"source": common.MapStr{
+						"address": metadata.RemoteAddr.String(),
+					},
+				},
 			},
 		}
 		forwarder.Send(e)

diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json
@@ -12,6 +12,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/access/test/test.log", 
         "offset": 0, 
         "prospector.type": "log"
     }, 
@@ -38,6 +39,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/access/test/test.log", 
         "offset": 73, 
         "prospector.type": "log"
     }, 
@@ -50,6 +52,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/access/test/test.log", 
         "offset": 238, 
         "prospector.type": "log"
     }, 
@@ -75,6 +78,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "access", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/access/test/test.log", 
         "offset": 285, 
         "prospector.type": "log"
     }

diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json
@@ -8,6 +8,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/error/test/test.log", 
         "offset": 0, 
         "prospector.type": "log"
     }, 
@@ -21,6 +22,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/error/test/test.log", 
         "offset": 99, 
         "prospector.type": "log"
     }, 
@@ -36,6 +38,7 @@
         "fileset.module": "apache2", 
         "fileset.name": "error", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/apache2/error/test/test.log", 
         "offset": 229, 
         "prospector.type": "log"
     }

diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json
@@ -15,6 +15,7 @@
         "fileset.module": "auditd", 
         "fileset.name": "log", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", 
         "offset": 0, 
         "prospector.type": "log"
     }, 
@@ -50,6 +51,7 @@
         "fileset.module": "auditd", 
         "fileset.name": "log", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", 
         "offset": 174, 
         "prospector.type": "log"
     }

diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -10,6 +10,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", 
         "offset": 0, 
         "prospector.type": "log", 
@@ -27,6 +28,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", 
         "offset": 155, 
         "prospector.type": "log", 
@@ -45,6 +47,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", 
         "offset": 306, 
         "prospector.type": "log", 
@@ -61,6 +64,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", 
         "offset": 519, 
         "prospector.type": "log", 
@@ -77,6 +81,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", 
         "offset": 654, 
         "prospector.type": "log", 
@@ -95,6 +100,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", 
         "offset": 802, 
         "prospector.type": "log", 
@@ -113,6 +119,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "audit", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/audit/test/test.log", 
         "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", 
         "offset": 986, 
         "prospector.type": "log", 

diff --git a/filebeat/module/elasticsearch/gc/test/test.log-expected.json b/filebeat/module/elasticsearch/gc/test/test.log-expected.json
@@ -15,6 +15,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/gc/test/test.log", 
         "message": "2018-03-03T19:37:06.157+0500: 14597.826: [GC (CMS Initial Mark) [1 CMS-initial-mark: 131804K(174784K)] 142444K(253440K), 0.0021716 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", 
         "offset": 0, 
         "prospector.type": "log", 
@@ -29,6 +30,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/gc/test/test.log", 
         "message": "2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds", 
         "offset": 181, 
         "prospector.type": "log", 
@@ -57,6 +59,7 @@
         "fileset.module": "elasticsearch", 
         "fileset.name": "gc", 
         "input.type": "log", 
+        "log.file.path": "/Users/ruflin/Dev/gopath/src/github.com/elastic/beats/filebeat/module/elasticsearch/gc/test/test.log", 
         "message": "2018-06-30T16:35:26.632+0500: 224.671: [GC (CMS Final Remark) [YG occupancy: 113198 K (157248 K)]224.671: [Rescan (parallel) , 0.0148273 secs]224.686: [weak refs processing, 0.0003647 secs]224.687: [class unloading, 0.0188407 secs]224.705: [scrub symbol table, 0.0100207 secs]224.715: [scrub string table, 0.0005253 secs][1 CMS-remark: 277821K(349568K)] 391020K(506816K), 0.0457689 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]", 
         "offset": 339, 
         "prospector.type": "log",