From 8db5073c2b94df042dc9598a34bc018649ff00d6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 15 Nov 2019 08:27:34 -0800 Subject: [PATCH 1/3] Fixing node name to use `DATA` grok pattern Previously, the node name field was being parsed using the `WORD` grok patterns, which does not allow for characters such as `-` in the node name. Such characters are acceptable in Elasticsearch node names, so this PR fixes the grok expression to try and parse the node name using the more-accepting `DATA` grok pattern. --- .../module/elasticsearch/slowlog/ingest/pipeline-plaintext.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json index f582bdbdf60..ae88869d0c4 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -9,7 +9,7 @@ "INDEXNAME": "[a-zA-Z0-9_.-]*" }, "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{WORD:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" + "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})?\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" ] } }, From ab2ca5b28b9a00e775cbad754abbfa0b82372b19 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 15 Nov 2019 08:30:51 -0800 Subject: [PATCH 2/3] Adding sample log entry --- .../elasticsearch/slowlog/test/test.log | 1 + .../slowlog/test/test.log-expected.json | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log b/filebeat/module/elasticsearch/slowlog/test/test.log index 3d6d1ebae79..52cbd3741ed 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log +++ b/filebeat/module/elasticsearch/slowlog/test/test.log @@ -35,3 +35,4 @@ "name":"Rados-MacBook-Pro.local" } }] +[2019-11-14T21:18:40,269][TRACE][index.search.slowlog.query] [exp-data-elasticsearc-2] [exp_v3_1_current][3] took[516.4ms], took_millis[516], types[encounter], stats[], search_type[QUERY_THEN_FETCH], total_shards[10], source[{"size":1000,"query":{"constant_score":{"filter":{"bool":{"must":[{"bool":{"should":[{"nested":{"query":{"constant_score":{"filter":{"bool":{"must":[{"term":{"diagnosis.dx_rank":{"value":1,"boost":1.0}}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"path":"diagnosis","ignore_unmapped":true,"score_mode":"avg","boost":1.0}},{"nested":{"query":{"constant_score":{"filter":{"bool":{"must":[{"term":{"procedure.px_rank":{"value":1,"boost":1.0}}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"path":"procedure","ignore_unmapped":true,"score_mode":"avg","boost":1.0}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}}],"must_not":[{"exists":{"field":"primary_px_key","boost":1.0}}],"disable_coord":false,"adjust_pure_negative":true,"boost":1.0}},"boost":1.0}},"version":true,"sort":[{"_doc":{"order":"asc"}}]}] diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 4b534272ea5..55fb7a6c3b6 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -140,5 +140,28 @@ "log.offset": 4766, "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[", "service.type": "elasticsearch" + }, + { + "@timestamp": "2019-11-14T21:18:40.269-02:00", + "elasticsearch.index.name": "exp_v3_1_current", + "elasticsearch.node.name": "exp-data-elasticsearc-2", + "elasticsearch.shard.id": "3", + "elasticsearch.slowlog.logger": "index.search.slowlog.query", + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source_query": "{\"size\":1000,\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"diagnosis.dx_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"diagnosis\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}},{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"procedure.px_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"procedure\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}],\"must_not\":[{\"exists\":{\"field\":\"primary_px_key\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"version\":true,\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}", + "elasticsearch.slowlog.stats": "", + "elasticsearch.slowlog.took": "516.4ms", + "elasticsearch.slowlog.total_shards": 10, + "elasticsearch.slowlog.types": "encounter", + "event.dataset": "elasticsearch.slowlog", + "event.duration": 516000000, + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "fileset.name": "slowlog", + "input.type": "log", + "log.level": "TRACE", + "log.offset": 5638, + "message": "[2019-11-14T21:18:40,269][TRACE][index.search.slowlog.query] [exp-data-elasticsearc-2] [exp_v3_1_current][3] took[516.4ms], took_millis[516], types[encounter], stats[], search_type[QUERY_THEN_FETCH], total_shards[10], source[{\"size\":1000,\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"diagnosis.dx_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"diagnosis\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}},{\"nested\":{\"query\":{\"constant_score\":{\"filter\":{\"bool\":{\"must\":[{\"term\":{\"procedure.px_rank\":{\"value\":1,\"boost\":1.0}}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"path\":\"procedure\",\"ignore_unmapped\":true,\"score_mode\":\"avg\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}}],\"must_not\":[{\"exists\":{\"field\":\"primary_px_key\",\"boost\":1.0}}],\"disable_coord\":false,\"adjust_pure_negative\":true,\"boost\":1.0}},\"boost\":1.0}},\"version\":true,\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}]", + "service.type": "elasticsearch" } ] \ No newline at end of file From 67c83f0032e9cb706563817e9cd9cc4aa3ca7c3f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 15 Nov 2019 08:31:47 -0800 Subject: [PATCH 3/3] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cd8016e2ad9..43c267941c9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -191,6 +191,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix race condition in S3 input plugin. {pull}14359[14359] - Decode hex values in auditd module. {pull}14471[14471] - Fix handling multiline log entries in nginx module. {issue}14349[14349] {pull}14499[14499] +- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] *Heartbeat*