Convert Filebeat redis.log to ECS (#9315)

- Convert many fields under `redis.log.*` to ECS. Previous field names are now field aliases towards the new corresponding ECS field: - redis.log.level => log.level - redis.log.message => message - redis.log.pid => process.pid - Coerce PID to an int
elastic · Dec 19, 2018 · 2209c46 · 2209c46
1 parent 09b3bb7
commit 2209c46
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 40 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -69,6 +69,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 - Stop runners disabled by hints after previously being started. {pull}9305[9305]
 - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417]
 - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487]
+- Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -433,6 +433,20 @@
   to: user_agent.original
   alias: true
 
+## Redis module
+
+- from: redis.log.pid
+  to: process.pid
+  alias: true
+
+- from: redis.log.level
+  to: log.level
+  alias: true
+
+- from: redis.log.message
+  to: message
+  alias: true
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6798,43 +6798,40 @@ Redis log files
 
 
 
-*`redis.log.pid`*::
+*`redis.log.role`*::
 +
 --
-type: long
+type: keyword
 
-The process ID of the Redis server.
+The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.
 
 
 --
 
-*`redis.log.role`*::
+*`redis.log.pid`*::
 +
 --
-type: keyword
-
-The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child), or `sentinel`.
+type: alias
 
+alias to: process.pid
 
 --
 
 *`redis.log.level`*::
 +
 --
-type: keyword
-
-The log level. Can be one of `debug`, `verbose`, `notice`, or `warning`.
+type: alias
 
+alias to: log.level
 
 --
 
 *`redis.log.message`*::
 +
 --
-type: text
-
-The log message
+type: alias
 
+alias to: message
 
 --
 

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/redis/log/_meta/fields.yml b/filebeat/module/redis/log/_meta/fields.yml
@@ -3,20 +3,21 @@
   description: >
     Redis log files
   fields:
-    - name: pid
-      type: long
-      description: >
-        The process ID of the Redis server.
     - name: role
       type: keyword
       description: >
         The role of the Redis instance. Can be one of `master`, `slave`, `child` (for RDF/AOF writing child),
         or `sentinel`.
+
+    - name: pid
+      type: alias
+      path: process.pid
+      migration: true
     - name: level
-      type: keyword
-      description: >
-        The log level. Can be one of `debug`, `verbose`, `notice`, or `warning`.
+      type: alias
+      path: log.level
+      migration: true
     - name: message
-      type: text
-      description: >
-        The log message
+      type: alias
+      path: message
+      migration: true
diff --git a/filebeat/module/redis/log/ingest/pipeline.json b/filebeat/module/redis/log/ingest/pipeline.json
@@ -5,8 +5,8 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "(%{POSINT:redis.log.pid}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:redis.log.level} %{GREEDYDATA:redis.log.message}",
-          "%{POSINT:redis.log.pid}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:redis.log.message}"
+          "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:log.level} %{GREEDYDATA:message}",
+          "%{POSINT:process.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:message}"
         ],
         "pattern_definitions": {
           "CHAR": "[a-zA-Z]",
@@ -16,17 +16,13 @@
     }, {
       "script": {
         "lang": "painless",
-        "inline": "if (ctx.redis.log.level == '.') {\n          ctx.redis.log.level = 'debug';\n        } else if (ctx.redis.log.level == '-') {\n          ctx.redis.log.level = 'verbose';\n        } else if (ctx.redis.log.level == '*') {\n          ctx.redis.log.level = 'notice';\n        } else if (ctx.redis.log.level == '#') {\n          ctx.redis.log.level = 'warning';\n        }"
+        "inline": "if (ctx.log.level == '.') {\n          ctx.log.level = 'debug';\n        } else if (ctx.log.level == '-') {\n          ctx.log.level = 'verbose';\n        } else if (ctx.log.level == '*') {\n          ctx.log.level = 'notice';\n        } else if (ctx.log.level == '#') {\n          ctx.log.level = 'warning';\n        }"
       }
     }, {
       "script": {
         "lang": "painless",
         "inline": "if (ctx.redis.log.role == 'M') {\n          ctx.redis.log.role = 'master';\n        } else if (ctx.redis.log.role == 'S') {\n          ctx.redis.log.role = 'slave';\n        } else if (ctx.redis.log.role == 'C') {\n          ctx.redis.log.role = 'child';\n        } else if (ctx.redis.log.role == 'X') {\n          ctx.redis.log.role = 'sentinel';\n        }\n        "
       }
-    }, {
-      "remove": {
-        "field": "message"
-      }
     }, {
       "rename": {
         "field": "@timestamp",

diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json
@@ -4,37 +4,37 @@
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "notice", 
         "log.offset": 0, 
-        "redis.log.level": "notice", 
-        "redis.log.message": "Saving the final RDB snapshot before exiting.", 
-        "redis.log.pid": "98738", 
+        "message": "Saving the final RDB snapshot before exiting.", 
+        "process.pid": 98738, 
         "redis.log.role": "master"
     }, 
     {
         "@timestamp": "2018-05-30T10:05:20.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "debug", 
         "log.offset": 76, 
-        "redis.log.level": "debug", 
-        "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
+        "message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     }, 
     {
         "@timestamp": "2018-05-31T04:32:08.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
+        "log.level": "notice", 
         "log.offset": 165, 
-        "redis.log.level": "notice", 
-        "redis.log.message": "The server is now ready to accept connections on port 6379\""
+        "message": "The server is now ready to accept connections on port 6379\""
     }, 
     {
         "@timestamp": "2017-05-30T10:57:24.000Z", 
         "event.dataset": "log", 
         "event.module": "redis", 
         "input.type": "log", 
         "log.offset": 250, 
-        "redis.log.message": "Received SIGINT scheduling shutdown...", 
-        "redis.log.pid": "5092"
+        "message": "Received SIGINT scheduling shutdown...", 
+        "process.pid": 5092
     }
 ]