From 25b0b92cc8d3b2834e8cdc7f44765a3961ff1ed7 Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Wed, 25 Mar 2020 15:03:53 +0100 Subject: [PATCH 1/4] Allow a wider range of characters for logstash.log.pipeline_id Allow a wider range of characters for `logstash.log.pipeline_id`. The `NOTSPACE` should be fine, but for covering any case, `DATA` would be safer. Should solve https://github.com/elastic/beats/issues/17242 --- filebeat/module/logstash/log/ingest/pipeline-plain.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.yml b/filebeat/module/logstash/log/ingest/pipeline-plain.yml index ced54ba471b..6ec17b74073 100644 --- a/filebeat/module/logstash/log/ingest/pipeline-plain.yml +++ b/filebeat/module/logstash/log/ingest/pipeline-plain.yml @@ -13,7 +13,7 @@ processors: (.| )* patterns: - - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{WORD:logstash.log.pipeline_id}\] + - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\] %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\] %{GREEDYMULTILINE:message} From 9dd6c1a08597f399b10748fc180538b27a2a4e1a Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Wed, 25 Mar 2020 15:19:08 +0100 Subject: [PATCH 2/4] update tests for logstash plain 7.4 pipeline id --- .../logstash/log/test/logstash-plain-7.4.log | 1 + .../test/logstash-plain-7.4.log-expected.json | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log b/filebeat/module/logstash/log/test/logstash-plain-7.4.log index b0514986d25..36d24807bfe 100644 --- a/filebeat/module/logstash/log/test/logstash-plain-7.4.log +++ b/filebeat/module/logstash/log/test/logstash-plain-7.4.log @@ -1 +1,2 @@ [2019-11-20T19:04:48,468][WARN ][org.logstash.dissect.Dissector][the_pipeline_id] Dissector mapping, pattern not found {"field"=>"message", "pattern"=>"%{LogLineTimeStamp->}\t%{Healthy}\t%{Fatals}\t%{Errors}\t%{Warnings}\t%{TimeToBuildPatternsCache}\t%{CachedPatternsCount}\t%{MessagesEnqueued}\t%{DropMsgNoSubscribers}\t%{MessagesEnqueued}\t%{TotalDests}\t%{CycleProcTime}\t%{TimeSinceNap}\t%{QUtilPermilAvg}\t%{QUtilPermilMax}\t%{QUtilPermilCount}\t%{NotifierRequests}\t%{NotifierProcessedRequests}\t%{NotifierRequestsChangeDynamicSubs}\t%{NotifierSentRequestsChangeExtDynamicSubs}\t%{NotifierProcessedRequestsDropped}\t%{NotifierBadTargets}\t%{NotifierCycleTimeNetAvg}\t%{NotifierCycleTimeNetCount}\t%{NotifierUtilAvg->}", "event"=>{"fields"=>{"pipeline"=>"mypipeline", "indexprefix"=>"idx", "regid"=>"w", "env"=>"production"}, "beat"=>{"version"=>"6.8.3", "hostname"=>"myhostname", "name"=>"myname"}, "message"=>"msg", "tags"=>["production", "beats_input_codec_plain_applied"], "host"=>{"name"=>"myhostname"}}} +[2019-11-20T19:04:48,468][ERROR][logstash.outputs.elasticsearch][.monitoring-logstash] Encountered a retryable error. Will Retry with exponential backoff... diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json index bf3240e3984..f16ba0ee80b 100644 --- a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json +++ b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json @@ -14,5 +14,21 @@ "logstash.log.pipeline_id": "the_pipeline_id", "message": "Dissector mapping, pattern not found {\"field\"=>\"message\", \"pattern\"=>\"%{LogLineTimeStamp->}\\t%{Healthy}\\t%{Fatals}\\t%{Errors}\\t%{Warnings}\\t%{TimeToBuildPatternsCache}\\t%{CachedPatternsCount}\\t%{MessagesEnqueued}\\t%{DropMsgNoSubscribers}\\t%{MessagesEnqueued}\\t%{TotalDests}\\t%{CycleProcTime}\\t%{TimeSinceNap}\\t%{QUtilPermilAvg}\\t%{QUtilPermilMax}\\t%{QUtilPermilCount}\\t%{NotifierRequests}\\t%{NotifierProcessedRequests}\\t%{NotifierRequestsChangeDynamicSubs}\\t%{NotifierSentRequestsChangeExtDynamicSubs}\\t%{NotifierProcessedRequestsDropped}\\t%{NotifierBadTargets}\\t%{NotifierCycleTimeNetAvg}\\t%{NotifierCycleTimeNetCount}\\t%{NotifierUtilAvg->}\", \"event\"=>{\"fields\"=>{\"pipeline\"=>\"mypipeline\", \"indexprefix\"=>\"idx\", \"regid\"=>\"w\", \"env\"=>\"production\"}, \"beat\"=>{\"version\"=>\"6.8.3\", \"hostname\"=>\"myhostname\", \"name\"=>\"myname\"}, \"message\"=>\"msg\", \"tags\"=>[\"production\", \"beats_input_codec_plain_applied\"], \"host\"=>{\"name\"=>\"myhostname\"}}}", "service.type": "logstash" + }, + { + "@timestamp": "2019-11-20T19:04:48.468-02:00", + "event.dataset": "logstash.log", + "event.kind": "event", + "event.module": "logstash", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "ERROR", + "log.offset": 1, + "logstash.log.module": "logstash.outputs.elasticsearch", + "logstash.log.pipeline_id": ".monitoring-logstash", + "message": "Encountered a retryable error. Will Retry with exponential backoff...", + "service.type": "logstash" } -] \ No newline at end of file +] From 20b6f72b730053fbd3915d538fdb0366e1efe6bc Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Wed, 25 Mar 2020 19:44:35 +0100 Subject: [PATCH 3/4] Failed tests --- .../logstash/log/test/logstash-plain-7.4.log-expected.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json index f16ba0ee80b..b9b93ff3140 100644 --- a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json +++ b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json @@ -21,11 +21,11 @@ "event.kind": "event", "event.module": "logstash", "event.timezone": "-02:00", - "event.type": "info", + "event.type": "error", "fileset.name": "log", "input.type": "log", "log.level": "ERROR", - "log.offset": 1, + "log.offset": 1015, "logstash.log.module": "logstash.outputs.elasticsearch", "logstash.log.pipeline_id": ".monitoring-logstash", "message": "Encountered a retryable error. Will Retry with exponential backoff...", From 3f731f5e56e6d07cdeb81e1edd5c2d1c205603de Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Mon, 30 Mar 2020 18:22:56 +0200 Subject: [PATCH 4/4] update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a600ea12d1d..3e9e8a36ee5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -99,6 +99,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix default index pattern in IBM MQ filebeat dashboard. {pull}17146[17146] - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164] - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220] +- Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243] *Heartbeat*