diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a385332ed54..95b623f5e7e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Modify apache/error dataset to follow ECS. {pull}8963[8963] - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] +- Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] *Heartbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 942fc676c8d..2964d4b8de3 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -404,6 +404,36 @@ to: message alias: true +## Kibana module + +- from: kibana.log.meta.req.headers.referer + to: http.request.referrer + alias: true + +- from: kibana.log.meta.req.referer + to: http.request.referrer + alias: true + +- from: kibana.log.meta.req.headers.user-agent + to: user_agent.original + alias: true + +- from: kibana.log.meta.req.remoteAddress + to: source.address + alias: true + +- from: kibana.log.meta.req.url + to: url.original + alias: true + +- from: kibana.log.meta.meta.statusCode + to: http.response.status_code + alias: true + +- from: kibana.log.meta.method + to: http.request.method + alias: true + ## NGINX module - from: nginx.access.user_name diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4dec70fe19b..28a18b8bcc2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -5720,6 +5720,69 @@ type: object -- +*`kibana.log.kibana.log.meta.req.headers.referer`*:: ++ +-- +type: alias + +alias to: http.request.referrer + +-- + +*`kibana.log.kibana.log.meta.req.referer`*:: ++ +-- +type: alias + +alias to: http.request.referrer + +-- + +*`kibana.log.kibana.log.meta.req.headers.user-agent`*:: ++ +-- +type: alias + +alias to: user_agent.original + +-- + +*`kibana.log.kibana.log.meta.req.remoteAddress`*:: ++ +-- +type: alias + +alias to: source.address + +-- + +*`kibana.log.kibana.log.meta.req.url`*:: ++ +-- +type: alias + +alias to: url.original + +-- + +*`kibana.log.kibana.log.meta.statusCode`*:: ++ +-- +type: alias + +alias to: http.response.status_code + +-- + +*`kibana.log.kibana.log.meta.method`*:: ++ +-- +type: alias + +alias to: http.request.method + +-- + [[exported-fields-kubernetes-processor]] == Kubernetes fields diff --git a/filebeat/module/kibana/fields.go b/filebeat/module/kibana/fields.go index 3ef7bd487d8..59fdee6b6ac 100644 --- a/filebeat/module/kibana/fields.go +++ b/filebeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyskEFuwyAQRfc+xVf2yQFYdNNl1TNUkzBGFMJYMFbl21cxboUt2lVm+T9674szAi8GwV8p0QCo18gGpxqcBsByuWU/qZdk8DIA2F7jXewceQBGz9EWs3ZnJLpzQ3ycLhMbuCzztCUd6p7TsqK436wH+xNY743GQA8Iok9cLk15NLZWJVd2xY868PIl2R66fwasI+qfRXHOJ7fCL11tUVJ+nvd1zpmTVixk3Ib03XdW6qrl+sk3PVQ1/NiP+w4AAP//OeSYZw==" + return "eJzMlEHO2yAQhfc+xejfhwN4UanKsuoZookZY2oM7jCoyu0rG6dyCGnaKIuf5Uz43stjxgcY6dLCaM/osQEQK45a+MiFjwZAU+zYzmKDb+FLAwDbr+F70MlRA9Bbcjq2a+8AHifaEZcjl5laMBzSvFUq1FvOnuWC+VOrwR4C8/mG/YgLBJz1FNWuWSruVQVNvGlcpUe6/Aqsi95fDKwmcmYuGGO9WeGqKhsFhd6ne0zM5CVjIfSbkbr2RIJV6XD+QZ0UrVw81c1V+XkolAtGLVKK6acaCDVxVEw9MXFVHp3F8i1mlKGFQWReMImiZMQ9Y7KGMecjnOifnX0+R9esUiQ+oCFfPskzc8vF03pRBbbGenRvC2sKQl+1Zor1tXnsKobEHSmsXn7VUOLyrz0Nh907U1k2LsVj0PVtfjpBcQ4+0oY5dfecV0xNJEMoPyH/NdJVQmnldwAAAP//jzOwrA==" } diff --git a/filebeat/module/kibana/log/_meta/fields.yml b/filebeat/module/kibana/log/_meta/fields.yml index f7f87416490..9ef1c657806 100644 --- a/filebeat/module/kibana/log/_meta/fields.yml +++ b/filebeat/module/kibana/log/_meta/fields.yml @@ -14,3 +14,32 @@ - name: meta type: object object_type: keyword + + - name: kibana.log.meta.req.headers.referer + type: alias + path: http.request.referrer + migration: true + - name: kibana.log.meta.req.referer + type: alias + path: http.request.referrer + migration: true + - name: kibana.log.meta.req.headers.user-agent + type: alias + path: user_agent.original + migration: true + - name: kibana.log.meta.req.remoteAddress + type: alias + path: source.address + migration: true + - name: kibana.log.meta.req.url + type: alias + path: url.original + migration: true + - name: kibana.log.meta.statusCode + type: alias + path: http.response.status_code + migration: true + - name: kibana.log.meta.method + type: alias + path: http.request.method + migration: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index 2e4d42814a8..0822a0624c4 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -53,6 +53,14 @@ "ignore_missing": true } }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.kibana.log.meta?.res?.responseTime != null" + } + }, { "rename": { "field": "kibana.log.meta.res.responseTime", @@ -74,6 +82,62 @@ "ignore_missing": true } }, + + { + "rename": { + "field": "kibana.log.meta.req.headers.referer", + "target_field": "http.request.referrer", + "ignore_missing": true + } + }, + { + "rename": { + "field": "kibana.log.meta.req.headers.user-agent", + "target_field": "user_agent.original", + "ignore_missing": true + } + }, + { + "rename": { + "field": "kibana.log.meta.req.remoteAddress", + "target_field": "source.address", + "ignore_missing": true + } + }, + { + "set": { + "field": "source.ip", + "value": "{{source.address}}", + "if": "ctx.source?.address != null" + } + }, + { + "rename": { + "field": "kibana.log.meta.req.url", + "target_field": "url.original", + "ignore_missing": true + } + }, + + { + "remove": { + "field": "kibana.log.meta.req.referer", + "ignore_missing": true + } + }, + { + "remove": { + "field": "kibana.log.meta.statusCode", + "ignore_missing": true + } + }, + { + "remove": { + "field": "kibana.log.meta.method", + "ignore_missing": true + } + }, + { "date": { "field": "read_timestamp", diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index fbc7301d87e..77283f2539b 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -3,14 +3,15 @@ "@timestamp": "2018-05-09T10:57:55.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "kibana.log", + "event.duration": 26000000, "event.module": "kibana", "fileset.name": "log", "http.request.method": "get", + "http.request.referrer": "http://localhost:5601/app/kibana", "http.response.content_length": 9, "http.response.elapsed_time": 26, "http.response.status_code": 304, "input.type": "log", - "kibana.log.meta.method": "get", "kibana.log.meta.req.headers.accept": "*/*", "kibana.log.meta.req.headers.accept-encoding": "gzip, deflate, br", "kibana.log.meta.req.headers.accept-language": "en-US,en;q=0.9,de;q=0.8", @@ -19,13 +20,7 @@ "kibana.log.meta.req.headers.if-modified-since": "Thu, 03 May 2018 09:45:28 GMT", "kibana.log.meta.req.headers.if-none-match": "\"24234c1c81b3948758c1a0be8e5a65386ca94c52\"", "kibana.log.meta.req.headers.origin": "http://localhost:5601", - "kibana.log.meta.req.headers.referer": "http://localhost:5601/app/kibana", - "kibana.log.meta.req.headers.user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", - "kibana.log.meta.req.referer": "http://localhost:5601/app/kibana", - "kibana.log.meta.req.remoteAddress": "127.0.0.1", - "kibana.log.meta.req.url": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", "kibana.log.meta.req.userAgent": "127.0.0.1", - "kibana.log.meta.statusCode": 304, "kibana.log.meta.type": "response", "kibana.log.tags": [], "log.offset": 0, @@ -33,7 +28,11 @@ "process.pid": 69410, "service.name": [ "kibana" - ] + ], + "source.address": "127.0.0.1", + "source.ip": "127.0.0.1", + "url.original": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" }, { "@timestamp": "2018-05-09T10:59:12.000Z",