From 510e8dab4d7f2566ab83abf6eb3b6dca15a28819 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 30 Nov 2020 08:22:14 -0700 Subject: [PATCH] Drop pkt_dstaddr and pkt_srcaddr when equals to "-" (#22721) * Add painless script to remove all empty fields (cherry picked from commit 24a4da8f1c2cd1e04eb488dc97e458d620e19da2) --- CHANGELOG.next.asciidoc | 1 + .../module/aws/vpcflow/ingest/pipeline.yml | 26 +++++++++ .../test/custom-nat-gateway.log-expected.json | 1 - .../test/no-data-skip-data.log-expected.json | 2 - .../test/tcp-flag-sequence-skip-data.log | 5 ++ ...-flag-sequence-skip-data.log-expected.json | 58 +++++++++++++++++++ 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91aa59c4bef..27e74637e62 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377] +- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] *Heartbeat* diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index bd9b1d32769..a8a6e5ae726 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -28,6 +28,32 @@ processors: field: ["aws.vpcflow.start", "aws.vpcflow.end"] ignore_missing: true + - script: + lang: painless + ignore_failure: true + if: ctx?.aws != null + source: >- + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v instanceof String && v == "-"); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx.aws); + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json index d508bd63479..6b9e4382bb5 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json @@ -1,6 +1,5 @@ [ { - "aws.vpcflow.instance_id": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json index 22705d87101..e8224ee08b1 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.version": "2", @@ -27,7 +26,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.version": "2", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log new file mode 100644 index 00000000000..2ce24460ff9 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log @@ -0,0 +1,5 @@ +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA + +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json new file mode 100644 index 00000000000..b28207021b6 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "SKIPDATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 183, + "service.type": "aws", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "NODATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 526, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file