Fix PANW bad mapping of client/source and server/dest packets and byt…

…es (elastic#18525) (elastic#18533) PANW pipeline was mapping: Bytes sent to client/destination bytes. Bytes received to server/source bytes. Packets sent to server/destination packets. Packets received to client/source packets. All of these mappings are wrong. The correct is: Bytes sent to client/source bytes. Bytes received to server/destination bytes. Packets sent to client/source packets. Packets received to server/destination packets. Also adding some missing ECS mappings in the process. Fixes elastic#18522 (cherry picked from commit 9bae856)
leweafan · May 15, 2020 · 47218e8 · 47218e8
1 parent 38f064a
commit 47218e8
Show file tree

Hide file tree

Showing 8 changed files with 3,117 additions and 1,676 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -131,6 +131,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472]
 - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
 - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
+- Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/panw/panos/config/input.yml b/x-pack/filebeat/module/panw/panos/config/input.yml
@@ -50,41 +50,49 @@ processors:
         destination.ip: 8
         destination.address: 8
         source.nat.ip: 9
+        client.nat.ip: 9
         destination.nat.ip: 10
+        server.nat.ip: 10
         panw.panos.ruleset: 11
         client.user.name: 12
         source.user.name: 12
         server.user.name: 13
         destination.user.name: 13
         network.application: 14
         panw.panos.source.zone: 16
+        observer.ingress.zone: 16
         panw.panos.destination.zone: 17
+        observer.egress.zone: 17
         panw.panos.source.interface: 18
+        observer.ingress.interface.name: 18
         panw.panos.destination.interface: 19
+        observer.egress.interface.name: 19
         panw.panos.flow_id: 22
         client.port: 24
         source.port: 24
         destination.port: 25
         server.port: 25
         source.nat.port: 26
+        client.nat.port: 26
         destination.nat.port: 27
+        server.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
         event.outcome: 30
         network.bytes: 31
         client.bytes: 32
-        destination.bytes: 32
+        source.bytes: 32
         server.bytes: 33
-        source.bytes: 33
+        destination.bytes: 33
         network.packets: 34
         event.start: 35
         event.duration: 36
         panw.panos.url.category: 37
         panw.panos.sequence_number: 39
-        server.packets: 44
-        destination.packets: 44
-        client.packets: 45
-        source.packets: 45
+        client.packets: 44
+        source.packets: 44
+        server.packets: 45
+        destination.packets: 45
         observer.hostname: 52
 
   - extract_array:
@@ -103,24 +111,32 @@ processors:
         destination.ip: 8
         destination.address: 8
         source.nat.ip: 9
+        client.nat.ip: 9
         destination.nat.ip: 10
+        server.nat.ip: 10
         panw.panos.ruleset: 11
         client.user.name: 12
         source.user.name: 12
         server.user.name: 13
         destination.user.name: 13
         network.application: 14
         panw.panos.source.zone: 16
+        observer.ingress.zone: 16
         panw.panos.destination.zone: 17
+        observer.egress.zone: 17
         panw.panos.source.interface: 18
+        observer.ingress.interface.name: 18
         panw.panos.destination.interface: 19
+        observer.egress.interface.name: 19
         panw.panos.flow_id: 22
         client.port: 24
         source.port: 24
         destination.port: 25
         server.port: 25
         source.nat.port: 26
+        client.nat.port: 26
         destination.nat.port: 27
+        server.nat.port: 27
         _temp_.labels: 28
         network.transport: 29
         event.outcome: 30

diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -75,6 +75,8 @@ processors:
  - convert: { type: long, ignore_missing: true, field: panw.panos.sequence_number }
  - convert: { type: long, ignore_missing: true, field: source.nat.port }
  - convert: { type: long, ignore_missing: true, field: destination.nat.port }
+ - convert: { type: long, ignore_missing: true, field: client.nat.port }
+ - convert: { type: long, ignore_missing: true, field: server.nat.port }
 
 # Remove PCAP ID when zero (no packet capture).
  - remove:
@@ -397,6 +399,22 @@ processors:
        - _temp_
      ignore_missing: true
 
+# Remove NAT fields when translation was not done.
+ - remove:
+     field:
+       - source.nat.ip
+       - source.nat.port
+       - client.nat.ip
+       - client.nat.port
+     if: 'ctx?.source?.nat?.ip == "0.0.0.0" && ctx?.source?.nat?.port == 0'
+ - remove:
+     field:
+       - destination.nat.ip
+       - destination.nat.port
+       - server.nat.ip
+       - server.nat.port
+     if: 'ctx?.destination?.nat?.ip == "0.0.0.0" && ctx?.destination?.nat?.port == 0'
+
 on_failure:
   - set:
       field: "error.message"

diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json
@@ -3,13 +3,13 @@
         "@timestamp": "2012-04-10T04:39:56.000-02:00",
         "client.bytes": 78,
         "client.ip": "192.168.0.2",
-        "client.packets": 0,
+        "client.packets": 1,
         "client.port": 59309,
         "client.user.name": "crusher",
         "destination.address": "204.232.231.46",
         "destination.as.number": 27357,
         "destination.as.organization.name": "Rackspace Hosting",
-        "destination.bytes": 78,
+        "destination.bytes": 0,
         "destination.geo.city_name": "Fort Lauderdale",
         "destination.geo.continent_name": "North America",
         "destination.geo.country_iso_code": "US",
@@ -18,9 +18,7 @@
         "destination.geo.region_iso_code": "US-FL",
         "destination.geo.region_name": "Florida",
         "destination.ip": "204.232.231.46",
-        "destination.nat.ip": "0.0.0.0",
-        "destination.nat.port": 0,
-        "destination.packets": 1,
+        "destination.packets": 0,
         "destination.port": 80,
         "event.action": "flow_started",
         "event.category": "network_traffic",
@@ -43,6 +41,10 @@
         "network.packets": 1,
         "network.transport": "tcp",
         "network.type": "ipv4",
+        "observer.egress.interface.name": "ethernet1/1",
+        "observer.egress.zone": "untrust",
+        "observer.ingress.interface.name": "ethernet1/2",
+        "observer.ingress.zone": "trust",
         "observer.serial_number": "01606001116",
         "panw.panos.destination.interface": "ethernet1/1",
         "panw.panos.destination.nat.ip": "0.0.0.0",
@@ -64,15 +66,13 @@
         ],
         "server.bytes": 0,
         "server.ip": "204.232.231.46",
-        "server.packets": 1,
+        "server.packets": 0,
         "server.port": 80,
         "service.type": "panw",
         "source.address": "192.168.0.2",
-        "source.bytes": 0,
+        "source.bytes": 78,
         "source.ip": "192.168.0.2",
-        "source.nat.ip": "0.0.0.0",
-        "source.nat.port": 0,
-        "source.packets": 0,
+        "source.packets": 1,
         "source.port": 59309,
         "source.user.name": "crusher",
         "tags": [