Fix conditions for date processors without timezones (#13883)

Some ingest processors use `event.timezone` field as timezone for the date processor. In the case the field is set, the date processor was being run twice, one without timezone and another one with timezone. If the field is not set, the date processor was ignoring error. Change date processor parameters so only one is run and both are consistent with error handling.
elastic · Oct 3, 2019 · b380b9b · b380b9b
1 parent b28ac98
commit b380b9b
Show file tree

Hide file tree

Showing 10 changed files with 26 additions and 9 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -152,6 +152,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix incorrect field references in envoyproxy dashboard {issue}13420[13420] {pull}13421[13421]
 - Fixed early expiration of templates (Netflow v9 and IPFIX). {pull}13821[13821]
 - Fixed bad handling of sequence numbers when multiple observation domains were exported by a single device (Netflow V9 and IPFIX). {pull}13821[13821]
+- Fix conditions and error checking of date processors in ingest pipelines that use `event.timezone` to parse dates. {pull}13883[13883]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json
@@ -16,13 +16,21 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "apache.error.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "EEE MMM dd H:m:s yyyy",
                     "EEE MMM dd H:m:s.SSSSSS yyyy"
                 ],
-                "ignore_failure": true
+                "on_failure": [
+                    {
+                        "append": {
+                            "field": "error.message",
+                            "value": "{{ _ingest.on_failure_message }}"
+                        }
+                    }
+                ]
             }
         },
         {

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json
@@ -54,12 +54,13 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "elasticsearch.audit.@timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "yyyy-MM-dd'T'HH:mm:ss,SSS"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {

diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json
@@ -22,12 +22,13 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "elasticsearch.deprecation.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "yyyy-MM-dd'T'HH:mm:ss,SSS"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {

diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json
@@ -28,12 +28,13 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "elasticsearch.server.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "yyyy-MM-dd'T'HH:mm:ss,SSS"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json
@@ -23,12 +23,13 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "elasticsearch.slowlog.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "yyyy-MM-dd'T'HH:mm:ss,SSS"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {

diff --git a/filebeat/module/kafka/log/ingest/pipeline.json b/filebeat/module/kafka/log/ingest/pipeline.json
@@ -59,10 +59,11 @@
     },
     {
       "date": {
+        "if": "ctx.event.timezone == null",
         "field": "kafka.log.timestamp",
         "target_field": "@timestamp",
         "formats": ["yyyy-MM-dd HH:mm:ss,SSS"],
-        "ignore_failure": true
+        "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
       }
     },
     {

diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json
@@ -15,10 +15,11 @@
     }
   }, {
     "date": {
+      "if": "ctx.event.timezone == null",
       "field": "nginx.error.time",
       "target_field": "@timestamp",
       "formats": ["yyyy/MM/dd H:m:s"],
-      "ignore_failure": true
+      "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
     }
   }, {
     "date": {

diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json
@@ -41,14 +41,15 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "system.auth.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
                     "MMM  d HH:mm:ss",
                     "MMM dd HH:mm:ss",
                     "ISO8601"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {

diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json
@@ -29,6 +29,7 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "system.syslog.timestamp",
                 "target_field": "@timestamp",
                 "formats": [
@@ -37,7 +38,7 @@
                     "MMM d HH:mm:ss",
                     "ISO8601"
                 ],
-                "ignore_failure": true
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {