diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a86a3eba63e..36a2ef96432 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -102,6 +102,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164] - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220] - CEF: Fixed decoding errors caused by trailing spaces in messages. {pull}17253[17253] +- Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243] *Heartbeat* diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.yml b/filebeat/module/logstash/log/ingest/pipeline-plain.yml index ced54ba471b..6ec17b74073 100644 --- a/filebeat/module/logstash/log/ingest/pipeline-plain.yml +++ b/filebeat/module/logstash/log/ingest/pipeline-plain.yml @@ -13,7 +13,7 @@ processors: (.| )* patterns: - - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{WORD:logstash.log.pipeline_id}\] + - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\] %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\] %{GREEDYMULTILINE:message} diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log b/filebeat/module/logstash/log/test/logstash-plain-7.4.log index b0514986d25..36d24807bfe 100644 --- a/filebeat/module/logstash/log/test/logstash-plain-7.4.log +++ b/filebeat/module/logstash/log/test/logstash-plain-7.4.log @@ -1 +1,2 @@ [2019-11-20T19:04:48,468][WARN ][org.logstash.dissect.Dissector][the_pipeline_id] Dissector mapping, pattern not found {"field"=>"message", "pattern"=>"%{LogLineTimeStamp->}\t%{Healthy}\t%{Fatals}\t%{Errors}\t%{Warnings}\t%{TimeToBuildPatternsCache}\t%{CachedPatternsCount}\t%{MessagesEnqueued}\t%{DropMsgNoSubscribers}\t%{MessagesEnqueued}\t%{TotalDests}\t%{CycleProcTime}\t%{TimeSinceNap}\t%{QUtilPermilAvg}\t%{QUtilPermilMax}\t%{QUtilPermilCount}\t%{NotifierRequests}\t%{NotifierProcessedRequests}\t%{NotifierRequestsChangeDynamicSubs}\t%{NotifierSentRequestsChangeExtDynamicSubs}\t%{NotifierProcessedRequestsDropped}\t%{NotifierBadTargets}\t%{NotifierCycleTimeNetAvg}\t%{NotifierCycleTimeNetCount}\t%{NotifierUtilAvg->}", "event"=>{"fields"=>{"pipeline"=>"mypipeline", "indexprefix"=>"idx", "regid"=>"w", "env"=>"production"}, "beat"=>{"version"=>"6.8.3", "hostname"=>"myhostname", "name"=>"myname"}, "message"=>"msg", "tags"=>["production", "beats_input_codec_plain_applied"], "host"=>{"name"=>"myhostname"}}} +[2019-11-20T19:04:48,468][ERROR][logstash.outputs.elasticsearch][.monitoring-logstash] Encountered a retryable error. Will Retry with exponential backoff... diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json index bf3240e3984..b9b93ff3140 100644 --- a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json +++ b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json @@ -14,5 +14,21 @@ "logstash.log.pipeline_id": "the_pipeline_id", "message": "Dissector mapping, pattern not found {\"field\"=>\"message\", \"pattern\"=>\"%{LogLineTimeStamp->}\\t%{Healthy}\\t%{Fatals}\\t%{Errors}\\t%{Warnings}\\t%{TimeToBuildPatternsCache}\\t%{CachedPatternsCount}\\t%{MessagesEnqueued}\\t%{DropMsgNoSubscribers}\\t%{MessagesEnqueued}\\t%{TotalDests}\\t%{CycleProcTime}\\t%{TimeSinceNap}\\t%{QUtilPermilAvg}\\t%{QUtilPermilMax}\\t%{QUtilPermilCount}\\t%{NotifierRequests}\\t%{NotifierProcessedRequests}\\t%{NotifierRequestsChangeDynamicSubs}\\t%{NotifierSentRequestsChangeExtDynamicSubs}\\t%{NotifierProcessedRequestsDropped}\\t%{NotifierBadTargets}\\t%{NotifierCycleTimeNetAvg}\\t%{NotifierCycleTimeNetCount}\\t%{NotifierUtilAvg->}\", \"event\"=>{\"fields\"=>{\"pipeline\"=>\"mypipeline\", \"indexprefix\"=>\"idx\", \"regid\"=>\"w\", \"env\"=>\"production\"}, \"beat\"=>{\"version\"=>\"6.8.3\", \"hostname\"=>\"myhostname\", \"name\"=>\"myname\"}, \"message\"=>\"msg\", \"tags\"=>[\"production\", \"beats_input_codec_plain_applied\"], \"host\"=>{\"name\"=>\"myhostname\"}}}", "service.type": "logstash" + }, + { + "@timestamp": "2019-11-20T19:04:48.468-02:00", + "event.dataset": "logstash.log", + "event.kind": "event", + "event.module": "logstash", + "event.timezone": "-02:00", + "event.type": "error", + "fileset.name": "log", + "input.type": "log", + "log.level": "ERROR", + "log.offset": 1015, + "logstash.log.module": "logstash.outputs.elasticsearch", + "logstash.log.pipeline_id": ".monitoring-logstash", + "message": "Encountered a retryable error. Will Retry with exponential backoff...", + "service.type": "logstash" } -] \ No newline at end of file +]