diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8b139633ddc..8e751885aae 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -256,6 +256,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release Google Cloud module as GA. {pull}17511[17511] - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714] +- Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729] *Heartbeat* diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 4c9b2c266d3..174a429946a 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -12,12 +12,42 @@ processors: != 'index_search_slowlog' - remove: field: elasticsearch.slowlog.type +- dot_expander: + field: service.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.service.name + target_field: service.name + ignore_missing: true - rename: field: elasticsearch.slowlog.level target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.process.thread.name + target_field: process.thread.name + ignore_missing: true - rename: field: elasticsearch.slowlog.component target_field: elasticsearch.component + ignore_missing: true - dot_expander: field: cluster.name path: elasticsearch.slowlog @@ -68,9 +98,17 @@ processors: - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] - remove: field: elasticsearch.slowlog.message -- date: +- rename: + field: elasticsearch.slowlog.@timestamp + target_field: '@timestamp' + ignore_missing: true +- rename: field: elasticsearch.slowlog.timestamp target_field: '@timestamp' + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' formats: - ISO8601 ignore_failure: true diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index f04c42d5bc2..da9cc3a10fa 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -16,7 +16,10 @@ processors: if: ctx.first_char == '{' name: '{< IngestPipeline "pipeline-json" >}' - remove: - field: elasticsearch.slowlog.timestamp + field: + - elasticsearch.slowlog.timestamp + - elasticsearch.server.@timestamp + ignore_missing: true - script: lang: painless source: ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale) diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log new file mode 100644 index 00000000000..3704f88d189 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log @@ -0,0 +1,2 @@ +{"@timestamp":"2020-04-16T11:20:02.069Z", "log.level":"TRACE", "id":"5xy3gnEBmUEb0NJ1lijF", "message":"[test_index/M4fNwSWlTfek9m1SNL49Kg]", "source":"{\\\"f", "took":"15.1ms", "took_millis":"15" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#2]","log.logger":"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2020-04-16T11:20:02.777Z", "log.level":"TRACE", "id":"6By3gnEBmUEb0NJ1mSij", "message":"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]", "source":"{\\\"field\\\":123}", "took":"10.4ms", "took_millis":"10" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#4]","log.logger":"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json new file mode 100644 index 00000000000..cce5652340a --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2020-04-16T11:20:02.069Z", + "elasticsearch.cluster.name": "integTest", + "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", + "elasticsearch.index.id": "M4fNwSWlTfek9m1SNL49Kg", + "elasticsearch.index.name": "test_index", + "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", + "elasticsearch.node.name": "integTest-0", + "elasticsearch.slowlog.id": "5xy3gnEBmUEb0NJ1lijF", + "elasticsearch.slowlog.source": "{\\\"f", + "elasticsearch.slowlog.took": "15.1ms", + "event.category": "database", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 15000000, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "input.type": "log", + "log.level": "TRACE", + "log.logger": "index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg", + "log.offset": 0, + "message": "{\"@timestamp\":\"2020-04-16T11:20:02.069Z\", \"log.level\":\"TRACE\", \"id\":\"5xy3gnEBmUEb0NJ1lijF\", \"message\":\"[test_index/M4fNwSWlTfek9m1SNL49Kg]\", \"source\":\"{\\\\\\\"f\", \"took\":\"15.1ms\", \"took_millis\":\"15\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#2]\",\"log.logger\":\"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", + "process.thread.name": "elasticsearch[integTest-0][write][T#2]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-16T11:20:02.777Z", + "elasticsearch.cluster.name": "integTest", + "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", + "elasticsearch.index.id": "Jsz7IUYMQ9ubo2ahiMgCbQ", + "elasticsearch.index.name": "test_index", + "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", + "elasticsearch.node.name": "integTest-0", + "elasticsearch.slowlog.id": "6By3gnEBmUEb0NJ1mSij", + "elasticsearch.slowlog.source": "{\\\"field\\\":123}", + "elasticsearch.slowlog.took": "10.4ms", + "event.category": "database", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 10000000, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "input.type": "log", + "log.level": "TRACE", + "log.logger": "index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ", + "log.offset": 514, + "message": "{\"@timestamp\":\"2020-04-16T11:20:02.777Z\", \"log.level\":\"TRACE\", \"id\":\"6By3gnEBmUEb0NJ1mSij\", \"message\":\"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]\", \"source\":\"{\\\\\\\"field\\\\\\\":123}\", \"took\":\"10.4ms\", \"took_millis\":\"10\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#4]\",\"log.logger\":\"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", + "process.thread.name": "elasticsearch[integTest-0][write][T#4]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + } +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log new file mode 100644 index 00000000000..b817ea08fe7 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log @@ -0,0 +1,2 @@ +{"@timestamp":"2020-04-16T11:20:02.828Z", "log.level":"TRACE", "id":"null", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"10ms", "took_millis":"10", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#1]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2020-04-16T11:20:02.839Z", "log.level":"TRACE", "id":"my-identifier", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"76.4micros", "took_millis":"0", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#3]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json new file mode 100644 index 00000000000..39cd0679087 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json @@ -0,0 +1,66 @@ +[ + { + "@timestamp": "2020-04-16T11:20:02.828Z", + "elasticsearch.cluster.name": "integTest", + "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", + "elasticsearch.index.name": "test_index", + "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", + "elasticsearch.node.name": "integTest-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.id": "null", + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.took": "10ms", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.slowlog.total_shards": "1", + "event.category": "database", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 10000000, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "input.type": "log", + "log.level": "TRACE", + "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", + "log.offset": 0, + "message": "{\"@timestamp\":\"2020-04-16T11:20:02.828Z\", \"log.level\":\"TRACE\", \"id\":\"null\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"10ms\", \"took_millis\":\"10\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#1]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", + "process.thread.name": "elasticsearch[integTest-0][search][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-16T11:20:02.839Z", + "elasticsearch.cluster.name": "integTest", + "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", + "elasticsearch.index.name": "test_index", + "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", + "elasticsearch.node.name": "integTest-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.id": "my-identifier", + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.took": "76.4micros", + "elasticsearch.slowlog.total_hits": "0 hits", + "elasticsearch.slowlog.total_shards": "1", + "event.category": "database", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 0, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "input.type": "log", + "log.level": "TRACE", + "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", + "log.offset": 613, + "message": "{\"@timestamp\":\"2020-04-16T11:20:02.839Z\", \"log.level\":\"TRACE\", \"id\":\"my-identifier\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"76.4micros\", \"took_millis\":\"0\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#3]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", + "process.thread.name": "elasticsearch[integTest-0][search][T#3]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + } +] \ No newline at end of file