[7.7] Allow a wider range of characters for logstash.log.pipeline_id (#…

…17243) (#17357) * Allow a wider range of characters for logstash.log.pipeline_id (#17243) * Allow a wider range of characters for logstash.log.pipeline_id Allow a wider range of characters for `logstash.log.pipeline_id`. The `NOTSPACE` should be fine, but for covering any case, `DATA` would be safer. Should solve #17242 * update tests for logstash plain 7.4 pipeline id * Failed tests * update changelog * CHANGELOG grooming Co-authored-by: Luca Belluccini <luca.belluccini@elastic.co>
elastic · Mar 31, 2020 · cdc6abf · cdc6abf
1 parent 4d6bcfe
commit cdc6abf
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix default index pattern in IBM MQ filebeat dashboard. {pull}17146[17146]
 - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164]
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
+- Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243]
 
 *Heartbeat*
 

diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.yml b/filebeat/module/logstash/log/ingest/pipeline-plain.yml
@@ -13,7 +13,7 @@ processors:
         (.|
         )*
     patterns:
-    - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{WORD:logstash.log.pipeline_id}\]
+    - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\]
       %{GREEDYMULTILINE:message}
     - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]
       %{GREEDYMULTILINE:message}

diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log b/filebeat/module/logstash/log/test/logstash-plain-7.4.log
@@ -1 +1,2 @@
 [2019-11-20T19:04:48,468][WARN ][org.logstash.dissect.Dissector][the_pipeline_id] Dissector mapping, pattern not found {"field"=>"message", "pattern"=>"%{LogLineTimeStamp->}\t%{Healthy}\t%{Fatals}\t%{Errors}\t%{Warnings}\t%{TimeToBuildPatternsCache}\t%{CachedPatternsCount}\t%{MessagesEnqueued}\t%{DropMsgNoSubscribers}\t%{MessagesEnqueued}\t%{TotalDests}\t%{CycleProcTime}\t%{TimeSinceNap}\t%{QUtilPermilAvg}\t%{QUtilPermilMax}\t%{QUtilPermilCount}\t%{NotifierRequests}\t%{NotifierProcessedRequests}\t%{NotifierRequestsChangeDynamicSubs}\t%{NotifierSentRequestsChangeExtDynamicSubs}\t%{NotifierProcessedRequestsDropped}\t%{NotifierBadTargets}\t%{NotifierCycleTimeNetAvg}\t%{NotifierCycleTimeNetCount}\t%{NotifierUtilAvg->}", "event"=>{"fields"=>{"pipeline"=>"mypipeline", "indexprefix"=>"idx", "regid"=>"w", "env"=>"production"}, "beat"=>{"version"=>"6.8.3", "hostname"=>"myhostname", "name"=>"myname"}, "message"=>"msg", "tags"=>["production", "beats_input_codec_plain_applied"], "host"=>{"name"=>"myhostname"}}}
+[2019-11-20T19:04:48,468][ERROR][logstash.outputs.elasticsearch][.monitoring-logstash] Encountered a retryable error. Will Retry with exponential backoff...
diff --git a/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain-7.4.log-expected.json
@@ -14,5 +14,21 @@
         "logstash.log.pipeline_id": "the_pipeline_id",
         "message": "Dissector mapping, pattern not found {\"field\"=>\"message\", \"pattern\"=>\"%{LogLineTimeStamp->}\\t%{Healthy}\\t%{Fatals}\\t%{Errors}\\t%{Warnings}\\t%{TimeToBuildPatternsCache}\\t%{CachedPatternsCount}\\t%{MessagesEnqueued}\\t%{DropMsgNoSubscribers}\\t%{MessagesEnqueued}\\t%{TotalDests}\\t%{CycleProcTime}\\t%{TimeSinceNap}\\t%{QUtilPermilAvg}\\t%{QUtilPermilMax}\\t%{QUtilPermilCount}\\t%{NotifierRequests}\\t%{NotifierProcessedRequests}\\t%{NotifierRequestsChangeDynamicSubs}\\t%{NotifierSentRequestsChangeExtDynamicSubs}\\t%{NotifierProcessedRequestsDropped}\\t%{NotifierBadTargets}\\t%{NotifierCycleTimeNetAvg}\\t%{NotifierCycleTimeNetCount}\\t%{NotifierUtilAvg->}\", \"event\"=>{\"fields\"=>{\"pipeline\"=>\"mypipeline\", \"indexprefix\"=>\"idx\", \"regid\"=>\"w\", \"env\"=>\"production\"}, \"beat\"=>{\"version\"=>\"6.8.3\", \"hostname\"=>\"myhostname\", \"name\"=>\"myname\"}, \"message\"=>\"msg\", \"tags\"=>[\"production\", \"beats_input_codec_plain_applied\"], \"host\"=>{\"name\"=>\"myhostname\"}}}",
         "service.type": "logstash"
+    },
+    {
+        "@timestamp": "2019-11-20T19:04:48.468-02:00",
+        "event.dataset": "logstash.log",
+        "event.kind": "event",
+        "event.module": "logstash",
+        "event.timezone": "-02:00",
+        "event.type": "error",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.level": "ERROR",
+        "log.offset": 1015,
+        "logstash.log.module": "logstash.outputs.elasticsearch",
+        "logstash.log.pipeline_id": ".monitoring-logstash",
+        "message": "Encountered a retryable error. Will Retry with exponential backoff...",
+        "service.type": "logstash"
     }
-]
+]