Convert Filebeat kibana.log to ECS (#9301)

- Convert many fields under `kibana.log.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field:
  - `kibana.log.meta.req.headers.referer` => `http.request.referrer`
  - `kibana.log.meta.req.headers.user-agent` => `user_agent.original`
  - `kibana.log.meta.req.remoteAddress` => `source.ip`
  - `kibana.log.meta.req.url` => `url.original`
- Duplicate fields were removed, and are now aliased as well:
  - `kibana.log.meta.req.referer` => `http.request.referrer`
  - `kibana.log.meta.statusCode` => `http.response.status_code` (already existed as the ECS compliant field)
  - `kibana.log.meta.method` => `http.request.method` (already existed as the ECS compliant field)
- Lowercase `http.request.method` field
- Compute `event.duration` (in ns), based on `http.response.elapsed_time` (in ms)

CHANGELOG.asciidoc

@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Modify apache/error dataset to follow ECS. {pull}8963[8963]
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
+- Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301]
 
 *Heartbeat*
 

dev-tools/ecs-migration.yml

@@ -404,6 +404,36 @@
   to: message
   alias: true
 
+## Kibana module
+
+- from: kibana.log.meta.req.headers.referer
+  to: http.request.referrer
+  alias: true
+
+- from: kibana.log.meta.req.referer
+  to: http.request.referrer
+  alias: true
+
+- from: kibana.log.meta.req.headers.user-agent
+  to: user_agent.original
+  alias: true
+
+- from: kibana.log.meta.req.remoteAddress
+  to: source.address
+  alias: true
+
+- from: kibana.log.meta.req.url
+  to: url.original
+  alias: true
+
+- from: kibana.log.meta.meta.statusCode
+  to: http.response.status_code
+  alias: true
+
+- from: kibana.log.meta.method
+  to: http.request.method
+  alias: true
+
 ## NGINX module
 
 - from: nginx.access.user_name

filebeat/docs/fields.asciidoc

@@ -5720,6 +5720,69 @@ type: object
 
 --
 
+*`kibana.log.kibana.log.meta.req.headers.referer`*::
++
+--
+type: alias
+
+alias to: http.request.referrer
+
+--
+
+*`kibana.log.kibana.log.meta.req.referer`*::
++
+--
+type: alias
+
+alias to: http.request.referrer
+
+--
+
+*`kibana.log.kibana.log.meta.req.headers.user-agent`*::
++
+--
+type: alias
+
+alias to: user_agent.original
+
+--
+
+*`kibana.log.kibana.log.meta.req.remoteAddress`*::
++
+--
+type: alias
+
+alias to: source.address
+
+--
+
+*`kibana.log.kibana.log.meta.req.url`*::
++
+--
+type: alias
+
+alias to: url.original
+
+--
+
+*`kibana.log.kibana.log.meta.statusCode`*::
++
+--
+type: alias
+
+alias to: http.response.status_code
+
+--
+
+*`kibana.log.kibana.log.meta.method`*::
++
+--
+type: alias
+
+alias to: http.request.method
+
+--
+
 [[exported-fields-kubernetes-processor]]
 == Kubernetes fields
 

filebeat/module/kibana/log/_meta/fields.yml

@@ -14,3 +14,32 @@
     - name: meta
       type: object
       object_type: keyword
+
+    - name: kibana.log.meta.req.headers.referer
+      type: alias
+      path: http.request.referrer
+      migration: true
+    - name: kibana.log.meta.req.referer
+      type: alias
+      path: http.request.referrer
+      migration: true
+    - name: kibana.log.meta.req.headers.user-agent
+      type: alias
+      path: user_agent.original
+      migration: true
+    - name: kibana.log.meta.req.remoteAddress
+      type: alias
+      path: source.address
+      migration: true
+    - name: kibana.log.meta.req.url
+      type: alias
+      path: url.original
+      migration: true
+    - name: kibana.log.meta.statusCode
+      type: alias
+      path: http.response.status_code
+      migration: true
+    - name: kibana.log.meta.method
+      type: alias
+      path: http.request.method
+      migration: true

filebeat/module/kibana/log/ingest/pipeline.json

@@ -53,6 +53,14 @@
                 "ignore_missing": true
             }
         },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * params.scale)",
+                "params": { "scale": 1000000 },
+                "if": "ctx.kibana.log.meta?.res?.responseTime != null"
+            }
+        },
         {
             "rename": {
                 "field": "kibana.log.meta.res.responseTime",
@@ -74,6 +82,62 @@
                 "ignore_missing": true
             }
         },
+
+        {
+            "rename": {
+                "field": "kibana.log.meta.req.headers.referer",
+                "target_field": "http.request.referrer",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.req.headers.user-agent",
+                "target_field": "user_agent.original",
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.req.remoteAddress",
+                "target_field": "source.address",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "field": "source.ip",
+                "value": "{{source.address}}",
+                "if": "ctx.source?.address != null"
+            }
+        },
+        {
+            "rename": {
+                "field": "kibana.log.meta.req.url",
+                "target_field": "url.original",
+                "ignore_missing": true
+            }
+        },
+
+        {
+            "remove": {
+                "field": "kibana.log.meta.req.referer",
+                "ignore_missing": true
+            }
+        },
+        {
+            "remove": {
+                "field": "kibana.log.meta.statusCode",
+                "ignore_missing": true
+            }
+        },
+        {
+            "remove": {
+                "field": "kibana.log.meta.method",
+                "ignore_missing": true
+            }
+        },
+
         {
             "date": {
                 "field": "read_timestamp",

filebeat/module/kibana/log/test/test.log-expected.json

@@ -3,14 +3,15 @@
         "@timestamp": "2018-05-09T10:57:55.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "kibana.log",
+        "event.duration": 26000000,
         "event.module": "kibana",
         "fileset.name": "log",
         "http.request.method": "get",
+        "http.request.referrer": "http://localhost:5601/app/kibana",
         "http.response.content_length": 9,
         "http.response.elapsed_time": 26,
         "http.response.status_code": 304,
         "input.type": "log",
-        "kibana.log.meta.method": "get",
         "kibana.log.meta.req.headers.accept": "*/*",
         "kibana.log.meta.req.headers.accept-encoding": "gzip, deflate, br",
         "kibana.log.meta.req.headers.accept-language": "en-US,en;q=0.9,de;q=0.8",
@@ -19,21 +20,19 @@
         "kibana.log.meta.req.headers.if-modified-since": "Thu, 03 May 2018 09:45:28 GMT",
         "kibana.log.meta.req.headers.if-none-match": "\"24234c1c81b3948758c1a0be8e5a65386ca94c52\"",
         "kibana.log.meta.req.headers.origin": "http://localhost:5601",
-        "kibana.log.meta.req.headers.referer": "http://localhost:5601/app/kibana",
-        "kibana.log.meta.req.headers.user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
-        "kibana.log.meta.req.referer": "http://localhost:5601/app/kibana",
-        "kibana.log.meta.req.remoteAddress": "127.0.0.1",
-        "kibana.log.meta.req.url": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2",
         "kibana.log.meta.req.userAgent": "127.0.0.1",
-        "kibana.log.meta.statusCode": 304,
         "kibana.log.meta.type": "response",
         "kibana.log.tags": [],
         "log.offset": 0,
         "message": "GET /ui/fonts/open_sans/open_sans_v15_latin_600.woff2 304 26ms - 9.0B",
         "process.pid": 69410,
         "service.name": [
             "kibana"
-        ]
+        ],
+        "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
+        "url.original": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2",
+        "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
     },
     {
         "@timestamp": "2018-05-09T10:59:12.000Z",