Drop pkt_dstaddr and pkt_srcaddr when equals to "-" (elastic#22721)

* Add painless script to remove all empty fields (cherry picked from commit 24a4da8)
ChrsMark · Dec 1, 2020 · 510e8da · 510e8da
1 parent 711762b
commit 510e8da
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 3 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
 - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361]
 - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
+- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml
@@ -28,6 +28,32 @@ processors:
       field: ["aws.vpcflow.start", "aws.vpcflow.end"]
       ignore_missing: true
 
+  - script:
+      lang: painless
+      ignore_failure: true
+      if: ctx?.aws != null
+      source: >-
+        void handleMap(Map map) {
+          for (def x : map.values()) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+          }
+          map.values().removeIf(v -> v instanceof String && v == "-");
+        }
+        void handleList(List list) {
+          for (def x : list) {
+              if (x instanceof Map) {
+                  handleMap(x);
+              } else if (x instanceof List) {
+                  handleList(x);
+              }
+          }
+        }
+        handleMap(ctx.aws);
+
   # IP Geolocation Lookup
   - geoip:
       field: source.ip

diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json
@@ -1,6 +1,5 @@
 [
     {
-        "aws.vpcflow.instance_id": "-",
         "aws.vpcflow.interface_id": "eni-1235b8ca123456789",
         "aws.vpcflow.pkt_dstaddr": "203.0.113.5",
         "aws.vpcflow.pkt_srcaddr": "10.0.1.5",

diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json
@@ -2,7 +2,6 @@
     {
         "@timestamp": "2015-05-10T18:02:14.000Z",
         "aws.vpcflow.account_id": "123456789010",
-        "aws.vpcflow.action": "-",
         "aws.vpcflow.interface_id": "eni-1235b8ca123456789",
         "aws.vpcflow.log_status": "NODATA",
         "aws.vpcflow.version": "2",
@@ -27,7 +26,6 @@
     {
         "@timestamp": "2015-05-10T18:02:14.000Z",
         "aws.vpcflow.account_id": "123456789010",
-        "aws.vpcflow.action": "-",
         "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa",
         "aws.vpcflow.log_status": "SKIPDATA",
         "aws.vpcflow.version": "2",

diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log
@@ -0,0 +1,5 @@
+version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
+3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA
+
+version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
+3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA
diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json
@@ -0,0 +1,58 @@
+[
+    {
+        "@timestamp": "2019-08-26T19:48:53.000Z",
+        "aws.vpcflow.account_id": "123456789010",
+        "aws.vpcflow.instance_id": "i-01234567890123456",
+        "aws.vpcflow.interface_id": "eni-1235b8ca123456789",
+        "aws.vpcflow.log_status": "SKIPDATA",
+        "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
+        "aws.vpcflow.version": "3",
+        "aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
+        "cloud.account.id": "123456789010",
+        "cloud.instance.id": "i-01234567890123456",
+        "cloud.provider": "aws",
+        "event.category": "network_traffic",
+        "event.dataset": "aws.vpcflow",
+        "event.end": "2019-08-26T19:48:53.000Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA",
+        "event.start": "2019-08-26T19:47:55.000Z",
+        "event.type": "flow",
+        "fileset.name": "vpcflow",
+        "input.type": "log",
+        "log.offset": 183,
+        "service.type": "aws",
+        "tags": [
+            "forwarded"
+        ]
+    },
+    {
+        "@timestamp": "2019-08-26T19:48:53.000Z",
+        "aws.vpcflow.account_id": "123456789010",
+        "aws.vpcflow.instance_id": "i-01234567890123456",
+        "aws.vpcflow.interface_id": "eni-1235b8ca123456789",
+        "aws.vpcflow.log_status": "NODATA",
+        "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
+        "aws.vpcflow.version": "3",
+        "aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
+        "cloud.account.id": "123456789010",
+        "cloud.instance.id": "i-01234567890123456",
+        "cloud.provider": "aws",
+        "event.category": "network_traffic",
+        "event.dataset": "aws.vpcflow",
+        "event.end": "2019-08-26T19:48:53.000Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA",
+        "event.start": "2019-08-26T19:47:55.000Z",
+        "event.type": "flow",
+        "fileset.name": "vpcflow",
+        "input.type": "log",
+        "log.offset": 526,
+        "service.type": "aws",
+        "tags": [
+            "forwarded"
+        ]
+    }
+]