Convert Filebeat redis.log to ECS

CHANGELOG.asciidoc

@@ -69,6 +69,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 - Stop runners disabled by hints after previously being started. {pull}9305[9305]
 - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417]
 - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487]
+- Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315]
 
 *Heartbeat*
 

dev-tools/ecs-migration.yml

@@ -433,6 +433,20 @@
   to: user_agent.original
   alias: true
 
+## Redis module
+
+- from: redis.log.pid
+  to: process.pid
+  alias: true
+
+- from: redis.log.level
+  to: log.level
+  alias: true
+
+- from: redis.log.message
+  to: message
+  alias: true
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain

filebeat/docs/fields.asciidoc

@@ -6798,43 +6798,40 @@ Redis log files
 
 
 
-*`redis.log.pid`*::
+*`redis.log.role`*::
 +
 --
-type: long
+type: keyword
 
-The process ID of the Redis server.
+The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.
 
 
 --
 
-*`redis.log.role`*::
+*`redis.log.pid`*::
 +
 --
-type: keyword
-
-The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.
+type: alias
 
+alias to: process.pid
 
 --
 
 *`redis.log.level`*::
 +
 --
-type: keyword
-
-The log level. Can be one of `debug`, `verbose`, `notice`, or `warning`.
+type: alias
 
+alias to: log.level
 
 --
 
 *`redis.log.message`*::
 +
 --
-type: text
-
-The log message
+type: alias
 
+alias to: message
 
 --
 

filebeat/module/redis/log/_meta/fields.yml

@@ -3,20 +3,21 @@
   description: >
     Redis log files
   fields:
-    - name: pid
-      type: long
-      description: >
-        The process ID of the Redis server.
     - name: role
       type: keyword
       description: >
         The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child),
         or `sentinel`.
+
+    - name: pid
+      type: alias
+      path: process.pid
+      migration: true
     - name: level
-      type: keyword
-      description: >
-        The log level. Can be one of `debug`, `verbose`, `notice`, or `warning`.
+      type: alias
+      path: log.level
+      migration: true
     - name: message
-      type: text
-      description: >
-        The log message
+      type: alias
+      path: message
+      migration: true

filebeat/module/redis/log/ingest/pipeline.json

@@ -5,8 +5,8 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "(%{POSINT:redis.log.pid}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:redis.log.level} %{GREEDYDATA:redis.log.message}",
-          "%{POSINT:redis.log.pid}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:redis.log.message}"
+          "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:log.level} %{GREEDYDATA:message}",
+          "%{POSINT:process.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:message}"
         ],
         "pattern_definitions": {
           "CHAR": "[a-zA-Z]",
@@ -16,17 +16,13 @@
     }, {
       "script": {
         "lang": "painless",
-        "inline": "if (ctx.redis.log.level == '.') {\n          ctx.redis.log.level = 'debug';\n        } else if (ctx.redis.log.level == '-') {\n          ctx.redis.log.level = 'verbose';\n        } else if (ctx.redis.log.level == '*') {\n          ctx.redis.log.level = 'notice';\n        } else if (ctx.redis.log.level == '#') {\n          ctx.redis.log.level = 'warning';\n        }"
+        "inline": "if (ctx.log.level == '.') {\n          ctx.log.level = 'debug';\n        } else if (ctx.log.level == '-') {\n          ctx.log.level = 'verbose';\n        } else if (ctx.log.level == '*') {\n          ctx.log.level = 'notice';\n        } else if (ctx.log.level == '#') {\n          ctx.log.level = 'warning';\n        }"
       }
     }, {
       "script": {
         "lang": "painless",
         "inline": "if (ctx.redis.log.role == 'M') {\n          ctx.redis.log.role = 'master';\n        } else if (ctx.redis.log.role == 'S') {\n          ctx.redis.log.role = 'slave';\n        } else if (ctx.redis.log.role == 'C') {\n          ctx.redis.log.role = 'child';\n        } else if (ctx.redis.log.role == 'X') {\n          ctx.redis.log.role = 'sentinel';\n        }\n        "
       }
-    }, {
-      "remove": {
-        "field": "message"
-      }
     }, {
       "rename": {
         "field": "@timestamp",

filebeat/module/redis/log/test/test.log-expected.json

@@ -4,37 +4,37 @@
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "notice", 
         "log.offset": 0, 
-        "redis.log.level": "notice", 
-        "redis.log.message": "Saving the final RDB snapshot before exiting.", 
-        "redis.log.pid": "98738", 
+        "message": "Saving the final RDB snapshot before exiting.", 
+        "process.pid": 98738, 
         "redis.log.role": "master"
     }, 
     {
         "@timestamp": "2018-05-30T10:05:20.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "debug", 
         "log.offset": 76, 
-        "redis.log.level": "debug", 
-        "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
+        "message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     }, 
     {
         "@timestamp": "2018-05-31T04:32:08.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "notice", 
         "log.offset": 165, 
-        "redis.log.level": "notice", 
-        "redis.log.message": "The server is now ready to accept connections on port 6379\""
+        "message": "The server is now ready to accept connections on port 6379\""
     }, 
     {
         "@timestamp": "2017-05-30T10:57:24.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
         "log.offset": 250, 
-        "redis.log.message": "Received SIGINT scheduling shutdown...", 
-        "redis.log.pid": "5092"
+        "message": "Received SIGINT scheduling shutdown...", 
+        "process.pid": 5092
     }
 ]