Convert Filebeat logstash.* to ECS (#9935)

- Convert many fields under `kibana.log.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field: - logstash.log.level => log.level - logstash.log.message => message - logstash.slowlog.level => log.level - logstash.slowlog.took_in_nanos => event.duration - read_timestamp => event.created (not aliased yet, still used elsewhere) - logstash.slowlog.message was removed. Message was always the same, and the nested evvent it contains (in plain log) was already properly copied out to another field.
elastic · Jan 11, 2019 · c041e19 · c041e19
1 parent 6c1d73b
commit c041e19
Show file tree

Hide file tree

Showing 14 changed files with 114 additions and 99 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Modify apache/error dataset to follow ECS. {pull}8963[8963]
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
+- Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -434,6 +434,24 @@
   to: http.request.method
   alias: true
 
+## Logstash module
+
+- from: logstash.log.level
+  to: log.level
+  alias: true
+
+- from: logstash.log.message
+  to: message
+  alias: true
+
+- from: logstash.slowlog.level
+  to: log.level
+  alias: true
+
+- from: logstash.slowlog.took_in_nanos
+  to: event.duration
+  alias: true
+
 ## NGINX module
 
 - from: nginx.access.user_name

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6080,53 +6080,51 @@ Fields from the Logstash logs.
 
 
 
-*`logstash.log.message`*::
+*`logstash.log.module`*::
 +
 --
-type: text
+type: keyword
 
-Contains the un-parsed log message
+The module or class where the event originate.
 
 
 --
 
-*`logstash.log.level`*::
+*`logstash.log.thread`*::
 +
 --
-type: keyword
+type: text
 
-The log level of the message, this correspond to Log4j levels.
+Information about the running thread where the log originate.
 
 
 --
 
-*`logstash.log.module`*::
+*`logstash.log.log_event`*::
 +
 --
-type: keyword
+type: object
 
-The module or class where the event originate.
+key and value debugging information.
 
 
 --
 
-*`logstash.log.thread`*::
+*`logstash.log.message`*::
 +
 --
-type: text
-
-Information about the running thread where the log originate.
+type: alias
 
+alias to: message
 
 --
 
-*`logstash.log.log_event`*::
+*`logstash.log.level`*::
 +
 --
-type: object
-
-key and value debugging information.
+type: alias
 
+alias to: log.level
 
 --
 
@@ -6145,16 +6143,6 @@ type: text
 Contains the un-parsed log message
 
 
---
-
-*`logstash.slowlog.level`*::
-+
---
-type: keyword
-
-The log level of the message, this correspond to Log4j levels.
-
-
 --
 
 *`logstash.slowlog.module`*::
@@ -6217,33 +6205,41 @@ Execution time for the plugin in milliseconds.
 
 --
 
-*`logstash.slowlog.took_in_nanos`*::
+*`logstash.slowlog.plugin_params`*::
 +
 --
-type: long
+type: text
 
-Execution time for the plugin in nanoseconds.
+String value of the plugin configuration
 
 
 --
 
-*`logstash.slowlog.plugin_params`*::
+*`logstash.slowlog.plugin_params_object`*::
 +
 --
-type: text
+type: object
 
-String value of the plugin configuration
+key -> value of the configuration used by the plugin.
 
 
 --
 
-*`logstash.slowlog.plugin_params_object`*::
+*`logstash.slowlog.level`*::
 +
 --
-type: object
+type: alias
 
-key -> value of the configuration used by the plugin.
+alias to: log.level
+
+--
+
+*`logstash.slowlog.took_in_nanos`*::
++
+--
+type: alias
 
+alias to: event.duration
 
 --
 

diff --git a/filebeat/module/logstash/fields.go b/filebeat/module/logstash/fields.go
diff --git a/filebeat/module/logstash/log/_meta/fields.yml b/filebeat/module/logstash/log/_meta/fields.yml
@@ -4,14 +4,6 @@
   description: >
     Fields from the Logstash logs.
   fields:
-    - name: message
-      type: text
-      description: >
-        Contains the un-parsed log message
-    - name: level
-      type: keyword
-      description: >
-        The log level of the message, this correspond to Log4j levels.
     - name: module
       type: keyword
       description: >
@@ -25,3 +17,11 @@
       description: >
         key and value debugging information.
 
+    - name: message
+      type: alias
+      path: message
+      migration: true
+    - name: level
+      type: alias
+      path: log.level
+      migration: true
diff --git a/filebeat/module/logstash/log/ingest/pipeline-json.json b/filebeat/module/logstash/log/ingest/pipeline-json.json
@@ -1,5 +1,5 @@
 {
-    "description": "Pipeline for parsing logstash log logs",
+    "description": "Pipeline for parsing logstash logs",
     "on_failure": [
         {
             "set": {
@@ -18,7 +18,7 @@
         {
             "rename": {
                 "field": "@timestamp",
-                "target_field": "read_timestamp"
+                "target_field": "event.created"
             }
         },
         {
@@ -53,14 +53,20 @@
         {
             "rename": {
                 "field": "logstash.log.logEvent.message",
-                "target_field": "logstash.log.message"
+                "target_field": "message"
             }
         },
         {
             "rename": {
                 "field": "logstash.log.logEvent",
                 "target_field": "logstash.log.log_event"
             }
+        },
+        {
+            "rename": {
+                "field": "logstash.log.level",
+                "target_field": "log.level"
+            }
         }
     ]
 }
diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -13,24 +13,19 @@
             "grok": {
                 "field": "message",
                 "pattern_definitions": {
-                    "LOGSTASH_CLASS_MODULE": "[\\w\\.]+\\s*",
+                    "LOGSTASH_CLASS_MODULE": "[\\w\\.]+",
                     "LOGSTASH_LOGLEVEL": "INFO|ERROR|DEBUG|FATAL|WARN|TRACE",
                     "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:logstash.log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\] %{GREEDYMULTILINE:logstash.log.message}"
+                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\s*\\] %{GREEDYMULTILINE:message}"
                 ]
             }
         },
-        {
-            "remove": {
-                "field": "message"
-            }
-        },
         {
             "rename": {
                 "field": "@timestamp",
-                "target_field": "read_timestamp"
+                "target_field": "event.created"
             }
         },
         {

diff --git a/filebeat/module/logstash/log/test/logstash-json.log-expected.json b/filebeat/module/logstash/log/test/logstash-json.log-expected.json
@@ -9,8 +9,8 @@
         "log.flags": [
             "multiline"
         ],
+        "log.level": "INFO",
         "log.offset": 0,
-        "logstash.log.level": "INFO",
         "logstash.log.log_event.count": 1,
         "logstash.log.log_event.running_pipelines": [
             {
@@ -24,8 +24,8 @@
                 }
             }
         ],
-        "logstash.log.message": "Pipelines running",
         "logstash.log.module": "logstash.agent",
-        "logstash.log.thread": "Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6"
+        "logstash.log.thread": "Ruby-0-Thread-1: /Users/mat/work/elastic/releases/6.5.1/logstash/lib/bootstrap/environment.rb:6",
+        "message": "Pipelines running"
     }
 ]
diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
@@ -6,10 +6,10 @@
         "event.module": "logstash",
         "fileset.name": "log",
         "input.type": "log",
+        "log.level": "INFO",
         "log.offset": 0,
-        "logstash.log.level": "INFO",
-        "logstash.log.message": "Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}",
-        "logstash.log.module": "logstash.modules.scaffold"
+        "logstash.log.module": "logstash.modules.scaffold",
+        "message": "Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}"
     },
     {
         "@timestamp": "2017-11-20T03:55:00.318Z",
@@ -21,9 +21,9 @@
         "log.flags": [
             "multiline"
         ],
+        "log.level": "INFO",
         "log.offset": 175,
-        "logstash.log.level": "INFO",
-        "logstash.log.message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n",
-        "logstash.log.module": "logstash.inputs.jdbc     "
+        "logstash.log.module": "logstash.inputs.jdbc",
+        "message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n"
     }
 ]
diff --git a/filebeat/module/logstash/slowlog/_meta/fields.yml b/filebeat/module/logstash/slowlog/_meta/fields.yml
@@ -7,10 +7,6 @@
       type: text
       description: >
         Contains the un-parsed log message
-    - name: level
-      type: keyword
-      description: >
-        The log level of the message, this correspond to Log4j levels.
     - name: module
       type: keyword
       description: >
@@ -35,10 +31,6 @@
       type: long
       description: >
         Execution time for the plugin in milliseconds.
-    - name: took_in_nanos
-      type: long
-      description: >
-        Execution time for the plugin in nanoseconds.
     - name: plugin_params
       type: text
       description: >
@@ -48,7 +40,11 @@
       description: >
         key -> value of the configuration used by the plugin.
 
-
-
-
-
+    - name: level
+      type: alias
+      path: log.level
+      migration: true
+    - name: took_in_nanos
+      type: alias
+      path: event.duration
+      migration: true
diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline-json.json b/filebeat/module/logstash/slowlog/ingest/pipeline-json.json
@@ -1,5 +1,5 @@
 {
-    "description": "Pipeline for parsing logstash log logs",
+    "description": "Pipeline for parsing logstash slowlogs",
     "on_failure": [
         {
             "set": {
@@ -18,7 +18,7 @@
         {
             "rename": {
                 "field": "@timestamp",
-                "target_field": "read_timestamp"
+                "target_field": "event.created"
             }
         },
         {
@@ -51,7 +51,7 @@
         {
             "rename": {
                 "field": "logstash.slowlog.logEvent.took_in_nanos",
-                "target_field": "logstash.slowlog.took_in_nanos"
+                "target_field": "event.duration"
             }
         },
         {
@@ -82,6 +82,12 @@
                     "logstash.slowlog.logEvent"
                 ]
             }
+        },
+        {
+            "rename": {
+                "field": "logstash.slowlog.level",
+                "target_field": "log.level"
+            }
         }
     ]
 }