diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3eef5efb77f..40c40c282df 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -152,6 +152,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix incorrect field references in envoyproxy dashboard {issue}13420[13420] {pull}13421[13421] - Fixed early expiration of templates (Netflow v9 and IPFIX). {pull}13821[13821] - Fixed bad handling of sequence numbers when multiple observation domains were exported by a single device (Netflow V9 and IPFIX). {pull}13821[13821] +- Fix conditions and error checking of date processors in ingest pipelines that use `event.timezone` to parse dates. {pull}13883[13883] *Heartbeat* diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json index 8a1268780b1..89a6624aea9 100644 --- a/filebeat/module/apache/error/ingest/pipeline.json +++ b/filebeat/module/apache/error/ingest/pipeline.json @@ -16,13 +16,21 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "apache.error.timestamp", "target_field": "@timestamp", "formats": [ "EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy" ], - "ignore_failure": true + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] } }, { diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json index 448028cfa3b..345df18be13 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json @@ -54,12 +54,13 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "elasticsearch.audit.@timestamp", "target_field": "@timestamp", "formats": [ "yyyy-MM-dd'T'HH:mm:ss,SSS" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json index b3d95fad3c8..d9c4faada7c 100755 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json @@ -22,12 +22,13 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "elasticsearch.deprecation.timestamp", "target_field": "@timestamp", "formats": [ "yyyy-MM-dd'T'HH:mm:ss,SSS" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json index 0c9779acefb..b1752133d3a 100755 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json @@ -28,12 +28,13 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "elasticsearch.server.timestamp", "target_field": "@timestamp", "formats": [ "yyyy-MM-dd'T'HH:mm:ss,SSS" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json index e58407638e2..f582bdbdf60 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json @@ -23,12 +23,13 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "elasticsearch.slowlog.timestamp", "target_field": "@timestamp", "formats": [ "yyyy-MM-dd'T'HH:mm:ss,SSS" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/kafka/log/ingest/pipeline.json b/filebeat/module/kafka/log/ingest/pipeline.json index 0c6e44d39ca..6ba84de7634 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.json +++ b/filebeat/module/kafka/log/ingest/pipeline.json @@ -59,10 +59,11 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "kafka.log.timestamp", "target_field": "@timestamp", "formats": ["yyyy-MM-dd HH:mm:ss,SSS"], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json index 4c89300a92b..c66ac042650 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -15,10 +15,11 @@ } }, { "date": { + "if": "ctx.event.timezone == null", "field": "nginx.error.time", "target_field": "@timestamp", "formats": ["yyyy/MM/dd H:m:s"], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { "date": { diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json index debb3d064c2..8df0a77e582 100644 --- a/filebeat/module/system/auth/ingest/pipeline.json +++ b/filebeat/module/system/auth/ingest/pipeline.json @@ -41,6 +41,7 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "system.auth.timestamp", "target_field": "@timestamp", "formats": [ @@ -48,7 +49,7 @@ "MMM dd HH:mm:ss", "ISO8601" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, { diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json index 1bf54bc08e1..0c614b8a957 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.json +++ b/filebeat/module/system/syslog/ingest/pipeline.json @@ -29,6 +29,7 @@ }, { "date": { + "if": "ctx.event.timezone == null", "field": "system.syslog.timestamp", "target_field": "@timestamp", "formats": [ @@ -37,7 +38,7 @@ "MMM d HH:mm:ss", "ISO8601" ], - "ignore_failure": true + "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } }, {