Fix zeek pipelines to include millisecond precision (elastic#16766)

* Fix zeek pipelines to include millisecond precision * Add changelog entry (cherry picked from commit 2c310c0)
andrewstucki · Mar 3, 2020 · 88414db · 88414db
1 parent 19ca8b2
commit 88414db
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 30 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -44,6 +44,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 
 *Filebeat*
+
+- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766]
+- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
+- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
+- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
+- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906]
+- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
+- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
+- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
 - Fix a connection error in httpjson input. {pull}16123[16123]
 - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
 - Fix s3 input with cloudtrail fileset reading json file. {issue}16374[16374] {pull}16441[16441]

diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json
@@ -2,9 +2,20 @@
   "description": "Pipeline for normalizing Zeek files.log",
   "processors": [
     {
-      "script": {
-        "lang": "painless",
-        "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');"
+      "set": {
+        "field": "event.created",
+        "value": "{{_ingest.timestamp}}"
+      }
+    },
+    {
+      "date": {
+        "field": "zeek.files.ts",
+        "formats": ["UNIX"]
+      }
+    },
+    {
+      "remove": {
+        "field": "zeek.files.ts"
       }
     },
     {
@@ -37,10 +48,10 @@
       }
     }
   ],
-  "on_failure" : [{ 
-    "set" : { 
-      "field" : "error.message", 
-      "value" : "{{ _ingest.on_failure_message }}" 
-    } 
+  "on_failure": [{
+    "set": {
+      "field": "error.message",
+      "value": "{{ _ingest.on_failure_message }}"
+    }
   }]
-}
+}
diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": 1547688796000,
+        "@timestamp": "2019-01-17T01:33:16.636Z",
         "event.dataset": "zeek.files",
         "event.id": "C8I0zn3r9EPbfLgta6",
         "event.module": "zeek",
@@ -37,7 +37,7 @@
         "zeek.session_id": "C8I0zn3r9EPbfLgta6"
     },
     {
-        "@timestamp": 1547688801000,
+        "@timestamp": "2019-01-17T01:33:21.566Z",
         "event.dataset": "zeek.files",
         "event.id": "C6sjVo23iNApLnlAt6",
         "event.module": "zeek",

diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json
@@ -2,9 +2,20 @@
   "description": "Pipeline for normalizing Zeek http.log",
   "processors": [
     {
-      "script": {
-        "lang": "painless",
-        "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');"
+      "set": {
+        "field": "event.created",
+        "value": "{{_ingest.timestamp}}"
+      }
+    },
+    {
+      "date": {
+        "field": "zeek.http.ts",
+        "formats": ["UNIX"]
+      }
+    },
+    {
+      "remove": {
+        "field": "zeek.http.ts"
       }
     },
     {
@@ -103,10 +114,10 @@
       }
     }
   ],
-  "on_failure" : [{ 
-    "set" : { 
-      "field" : "error.message", 
-      "value" : "{{ _ingest.on_failure_message }}" 
-    } 
+  "on_failure": [{
+    "set": {
+      "field": "error.message",
+      "value": "{{ _ingest.on_failure_message }}"
+    }
   }]
 }
diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": 1547687130000,
+        "@timestamp": "2019-01-17T01:05:30.172Z",
         "destination.address": "17.253.5.203",
         "destination.as.number": 6185,
         "destination.as.organization.name": "Apple Inc.",

diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json
@@ -2,9 +2,20 @@
   "description": "Pipeline for normalizing Zeek notice.log",
   "processors": [
     {
-      "script": {
-        "lang": "painless",
-        "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');"
+      "set": {
+        "field": "event.created",
+        "value": "{{_ingest.timestamp}}"
+      }
+    },
+    {
+      "date": {
+        "field": "zeek.notice.ts",
+        "formats": ["UNIX"]
+      }
+    },
+    {
+      "remove": {
+        "field": "zeek.notice.ts"
       }
     },
     {
@@ -95,10 +106,10 @@
       }
     }
   ],
-  "on_failure" : [{ 
-    "set" : { 
-      "field" : "error.message", 
-      "value" : "{{ _ingest.on_failure_message }}" 
-    } 
+  "on_failure": [{
+    "set": {
+      "field": "error.message",
+      "value": "{{ _ingest.on_failure_message }}"
+    }
   }]
 }
diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": 1320435875000,
+        "@timestamp": "2011-11-04T19:44:35.879Z",
         "event.dataset": "zeek.notice",
         "event.module": "zeek",
         "fileset.name": "notice",
@@ -20,7 +20,7 @@
         "zeek.notice.suppress_for": 3600
     },
     {
-        "@timestamp": 1551393388000,
+        "@timestamp": "2019-02-28T22:36:28.426Z",
         "destination.address": "207.154.238.205",
         "destination.as.number": 14061,
         "destination.as.organization.name": "DigitalOcean, LLC",