Introduce log.source.address and log.file.path for 7.x compatiblity (#…

…9435)

Related to #8902 but adding the fields instead of replacing

CHANGELOG.asciidoc

@@ -99,6 +99,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 - Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768].
 - Allow to force CRI format parsing for better performance {pull}8424[8424]
 - Add event.dataset to module events. {pull}9457[9457]
+- Add field log.source.address and log.file.path to replace source. {pull}9435[9435]
 
 *Heartbeat*
 
@@ -127,6 +128,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
 *Affecting all Beats*
 
 *Filebeat*
+- Deprecate field source. Will be replaced by log.source.address and log.file.path in 7.0. {pull}9435[9435]
 
 *Heartbeat*
 

filebeat/_meta/fields.common.yml

@@ -116,6 +116,16 @@
       description: >
         This field contains the flags of the event.
 
+    - name: log.source.address
+      type: keyword
+      description: >
+        Log source address.
+
+    - name: log.file.path
+      type: keyword
+      description: >
+        Log source path.
+
     - name: event.created
       type: date
       description: >

filebeat/docs/fields.asciidoc

@@ -3126,6 +3126,26 @@ Logging level.
 This field contains the flags of the event.
 
 
+--
+
+*`log.source.address`*::
++
+--
+type: keyword
+
+Log source address.
+
+
+--
+
+*`log.file.path`*::
++
+--
+type: keyword
+
+Log source path.
+
+
 --
 
 *`event.created`*::

filebeat/input/log/harvester.go

@@ -310,6 +310,11 @@ func (h *Harvester) Run() error {
 			fields := common.MapStr{
 				"source": state.Source,
 				"offset": startingOffset, // Offset here is the offset before the starting char.
+				"log": common.MapStr{
+					"file": common.MapStr{
+						"path": state.Source,
+					},
+				},
 			}
 			fields.DeepUpdate(message.Fields)
 

filebeat/input/syslog/input.go

@@ -204,6 +204,11 @@ func createEvent(ev *event, metadata inputsource.NetworkMetadata, timezone *time
 	f := common.MapStr{
 		"message": strings.TrimRight(ev.Message(), "\n"),
 		"source":  metadata.RemoteAddr.String(),
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": metadata.RemoteAddr.String(),
+			},
+		},
 	}
 
 	syslog := common.MapStr{}

filebeat/input/syslog/input_test.go

@@ -55,6 +55,11 @@ func TestWhenPriorityIsSet(t *testing.T) {
 			"facility_label": "user-level",
 			"priority":       13,
 		},
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": "127.0.0.1",
+			},
+		},
 	}
 
 	assert.Equal(t, expected, event.Fields)
@@ -77,6 +82,11 @@ func TestWhenPriorityIsNotSet(t *testing.T) {
 		},
 		"event":  common.MapStr{},
 		"syslog": common.MapStr{},
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"address": "127.0.0.1",
+			},
+		},
 	}
 
 	assert.Equal(t, expected, event.Fields)

filebeat/input/tcp/input.go

@@ -126,6 +126,11 @@ func createEvent(raw []byte, metadata inputsource.NetworkMetadata) *util.Data {
 		Fields: common.MapStr{
 			"message": string(raw),
 			"source":  metadata.RemoteAddr.String(),
+			"log": common.MapStr{
+				"source": common.MapStr{
+					"address": metadata.RemoteAddr.String(),
+				},
+			},
 		},
 	}
 	return data

filebeat/input/udp/input.go

@@ -75,6 +75,11 @@ func NewInput(
 			Fields: common.MapStr{
 				"message": string(data),
 				"source":  metadata.RemoteAddr.String(),
+				"log": common.MapStr{
+					"source": common.MapStr{
+						"address": metadata.RemoteAddr.String(),
+					},
+				},
 			},
 		}
 		forwarder.Send(e)

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-12-11T08:08:07.894Z", 
+        "@timestamp": "2018-12-12T11:22:05.182Z", 
         "event.dataset": "icinga.startup", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
@@ -12,7 +12,7 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-12-11T08:08:07.894Z", 
+        "@timestamp": "2018-12-12T11:22:05.182Z", 
         "event.dataset": "icinga.startup", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 

filebeat/tests/system/test_json.py

@@ -41,7 +41,7 @@ def test_docker_logs_filtering(self):
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/*",
-            json=dict(message_key="log", keys_under_root=True),
+            json=dict(message_key="log"),
             exclude_lines=["windows"]
         )
 
@@ -59,10 +59,10 @@ def test_docker_logs_filtering(self):
         output = self.read_output()
         assert len(output) == 19
 
-        assert all("log" in o for o in output)
-        assert all("time" in o for o in output)
-        assert all(o["stream"] == "stdout" for o in output)
-        assert all("windows" not in o["log"] for o in output)
+        assert all("json.log" in o for o in output)
+        assert all("json.time" in o for o in output)
+        assert all(o["json.stream"] == "stdout" for o in output)
+        assert all("windows" not in o["json.log"] for o in output)
 
     def test_simple_json_overwrite(self):
         """

filebeat/tests/system/test_modules.py

@@ -194,7 +194,7 @@ def clean_keys(obj):
     # The create timestamps area always new
     time_keys = ["read_timestamp", "event.created"]
     # source path and beat.version can be different for each run
-    other_keys = ["source", "beat.version"]
+    other_keys = ["source", "log.file.path", "beat.version"]
 
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)