Allow a wider range of characters for logstash.log.pipeline_id (elast…

…ic#17243) * Allow a wider range of characters for logstash.log.pipeline_id Allow a wider range of characters for `logstash.log.pipeline_id`. The `NOTSPACE` should be fine, but for covering any case, `DATA` would be safer. Should solve elastic#17242 * update tests for logstash plain 7.4 pipeline id * Failed tests * update changelog
ycombinator · Apr 1, 2020 · f1c862e · f1c862e
1 parent e2c8b8d
commit f1c862e
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -49,6 +49,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add queue_url definition in manifest file for aws module. {pull}16640{16640}
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
 - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164]
+- CEF: Fixed decoding errors caused by trailing spaces in messages. {pull}17253[17253]
+- Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243]
 
 *Heartbeat*
 

diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -18,7 +18,7 @@
                     "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\s*\\]\\[%{WORD:logstash.log.pipeline_id}\\] %{GREEDYMULTILINE:message}",
+                    "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\s*\\]\\[%{NOTSPACE:logstash.log.pipeline_id}\\] %{GREEDYMULTILINE:message}",
                     "\\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\\]\\[%{LOGSTASH_LOGLEVEL:log.level}\\s?\\]\\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\\s*\\] %{GREEDYMULTILINE:message}"
                 ]
             }

diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log b/filebeat/module/logstash/log/test/logstash-plain-7.4.log
@@ -1 +1,2 @@
 [2019-11-20T19:04:48,468][WARN ][org.logstash.dissect.Dissector][the_pipeline_id] Dissector mapping, pattern not found {"field"=>"message", "pattern"=>"%{LogLineTimeStamp->}\t%{Healthy}\t%{Fatals}\t%{Errors}\t%{Warnings}\t%{TimeToBuildPatternsCache}\t%{CachedPatternsCount}\t%{MessagesEnqueued}\t%{DropMsgNoSubscribers}\t%{MessagesEnqueued}\t%{TotalDests}\t%{CycleProcTime}\t%{TimeSinceNap}\t%{QUtilPermilAvg}\t%{QUtilPermilMax}\t%{QUtilPermilCount}\t%{NotifierRequests}\t%{NotifierProcessedRequests}\t%{NotifierRequestsChangeDynamicSubs}\t%{NotifierSentRequestsChangeExtDynamicSubs}\t%{NotifierProcessedRequestsDropped}\t%{NotifierBadTargets}\t%{NotifierCycleTimeNetAvg}\t%{NotifierCycleTimeNetCount}\t%{NotifierUtilAvg->}", "event"=>{"fields"=>{"pipeline"=>"mypipeline", "indexprefix"=>"idx", "regid"=>"w", "env"=>"production"}, "beat"=>{"version"=>"6.8.3", "hostname"=>"myhostname", "name"=>"myname"}, "message"=>"msg", "tags"=>["production", "beats_input_codec_plain_applied"], "host"=>{"name"=>"myhostname"}}}
+[2019-11-20T19:04:48,468][ERROR][logstash.outputs.elasticsearch][.monitoring-logstash] Encountered a retryable error. Will Retry with exponential backoff...
diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json
@@ -12,5 +12,21 @@
         "logstash.log.pipeline_id": "the_pipeline_id",
         "message": "Dissector mapping, pattern not found {\"field\"=>\"message\", \"pattern\"=>\"%{LogLineTimeStamp->}\\t%{Healthy}\\t%{Fatals}\\t%{Errors}\\t%{Warnings}\\t%{TimeToBuildPatternsCache}\\t%{CachedPatternsCount}\\t%{MessagesEnqueued}\\t%{DropMsgNoSubscribers}\\t%{MessagesEnqueued}\\t%{TotalDests}\\t%{CycleProcTime}\\t%{TimeSinceNap}\\t%{QUtilPermilAvg}\\t%{QUtilPermilMax}\\t%{QUtilPermilCount}\\t%{NotifierRequests}\\t%{NotifierProcessedRequests}\\t%{NotifierRequestsChangeDynamicSubs}\\t%{NotifierSentRequestsChangeExtDynamicSubs}\\t%{NotifierProcessedRequestsDropped}\\t%{NotifierBadTargets}\\t%{NotifierCycleTimeNetAvg}\\t%{NotifierCycleTimeNetCount}\\t%{NotifierUtilAvg->}\", \"event\"=>{\"fields\"=>{\"pipeline\"=>\"mypipeline\", \"indexprefix\"=>\"idx\", \"regid\"=>\"w\", \"env\"=>\"production\"}, \"beat\"=>{\"version\"=>\"6.8.3\", \"hostname\"=>\"myhostname\", \"name\"=>\"myname\"}, \"message\"=>\"msg\", \"tags\"=>[\"production\", \"beats_input_codec_plain_applied\"], \"host\"=>{\"name\"=>\"myhostname\"}}}",
         "service.type": "logstash"
+    },
+    {
+        "@timestamp": "2019-11-20T19:04:48.468-02:00",
+        "event.dataset": "logstash.log",
+        "event.kind": "event",
+        "event.module": "logstash",
+        "event.timezone": "-02:00",
+        "event.type": "error",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "ERROR",
+        "log.offset": 1015,
+        "logstash.log.module": "logstash.outputs.elasticsearch",
+        "logstash.log.pipeline_id": ".monitoring-logstash",
+        "message": "Encountered a retryable error. Will Retry with exponential backoff...",
+        "service.type": "logstash"
     }
-]
+]