From 6fbfd5c3a8c638fa08113cd4bf174f89283bd3d2 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Wed, 2 Sep 2020 15:05:56 -0500 Subject: [PATCH] add type & sub_type for all events --- .../module/panw/panos/config/input.yml | 13 ++-- .../module/panw/panos/ingest/pipeline.yml | 66 +++++++++---------- .../test/pan_inc_other.log-expected.json | 66 +++++++++++++++++++ 3 files changed, 104 insertions(+), 41 deletions(-) diff --git a/x-pack/filebeat/module/panw/panos/config/input.yml b/x-pack/filebeat/module/panw/panos/config/input.yml index f268a325cd1..ed3d089bb28 100644 --- a/x-pack/filebeat/module/panw/panos/config/input.yml +++ b/x-pack/filebeat/module/panw/panos/config/input.yml @@ -35,24 +35,23 @@ processors: - extract_array: field: csv overwrite_keys: true + omit_empty: true mappings: event.created: 1 observer.serial_number: 2 - _temp_.message_type: 3 - _temp_.message_subtype: 4 + panw.panos.type: 3 + panw.panos.sub_type: 4 _temp_.generated_time: 6 - extract_array: when: equals: - _temp_.message_type: TRAFFIC + panw.panos.type: TRAFFIC field: csv overwrite_keys: true omit_empty: true fail_on_error: false mappings: - panw.panos.type: 3 - panw.panos.sub_type: 4 client.ip: 7 source.ip: 7 source.address: 7 @@ -109,14 +108,12 @@ processors: - extract_array: when: equals: - _temp_.message_type: THREAT + panw.panos.type: THREAT field: csv omit_empty: true overwrite_keys: true fail_on_error: false mappings: - panw.panos.type: 3 - panw.panos.sub_type: 4 client.ip: 7 source.ip: 7 source.address: 7 diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 6e4b7f64d61..412ddeb5c58 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -134,72 +134,72 @@ processors: - set: field: network.direction value: inbound - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "untrust" && ctx?.panw?.panos?.destination?.zone == "trust"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "untrust" && ctx?.panw?.panos?.destination?.zone == "trust"' - set: field: network.direction value: outbound - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "trust" && ctx?.panw?.panos?.destination?.zone == "untrust"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "trust" && ctx?.panw?.panos?.destination?.zone == "untrust"' - set: field: network.direction value: internal - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "trust" && ctx?.panw?.panos?.destination?.zone == "trust"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "trust" && ctx?.panw?.panos?.destination?.zone == "trust"' - set: field: network.direction value: external - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "untrust" && ctx?.panw?.panos?.destination?.zone == "untrust"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.panw?.panos?.source?.zone == "untrust" && ctx?.panw?.panos?.destination?.zone == "untrust"' - set: field: network.direction value: unknown - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ((ctx?.panw?.panos?.source?.zone != "trust" && ctx?.panw?.panos?.source?.zone != "untrust") || (ctx?.panw?.panos?.destination?.zone != "trust" && ctx?.panw?.panos?.destination?.zone != "untrust"))' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ((ctx?.panw?.panos?.source?.zone != "trust" && ctx?.panw?.panos?.source?.zone != "untrust") || (ctx?.panw?.panos?.destination?.zone != "trust" && ctx?.panw?.panos?.destination?.zone != "untrust"))' # Set network.direction from threat direction (Threat logs). - set: field: network.direction value: inbound - if: 'ctx?._temp_?.message_type == "THREAT" && (ctx?._temp_?.direction == "0" || ctx?._temp_?.direction == "client-to-server")' + if: 'ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "0" || ctx?._temp_?.direction == "client-to-server")' - set: field: network.direction value: outbound - if: 'ctx?._temp_?.message_type == "THREAT" && (ctx?._temp_?.direction == "1" || ctx?._temp_?.direction == "server-to-client")' + if: 'ctx?.panw?.panos?.type == "THREAT" && (ctx?._temp_?.direction == "1" || ctx?._temp_?.direction == "server-to-client")' - set: field: network.direction value: unknown - if: 'ctx?._temp_?.message_type == "THREAT" && ctx?.network?.direction == null' + if: 'ctx?.panw?.panos?.type == "THREAT" && ctx?.network?.direction == null' # Set network.type for TRAFFIC. - set: field: network.type value: 'ipv4' - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.labels?.ipv6_session == null' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session == null' - set: field: network.type value: 'ipv6' - if: 'ctx?._temp_?.message_type == "TRAFFIC" && ctx?.labels?.ipv6_session != null' + if: 'ctx?.panw?.panos?.type == "TRAFFIC" && ctx?.labels?.ipv6_session != null' # Set event.category depending on log type. - set: field: event.kind value: event - if: 'ctx?._temp_?.message_type == "TRAFFIC"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC"' - append: field: event.category value: - network_traffic - network - if: 'ctx?._temp_?.message_type == "TRAFFIC"' + if: 'ctx?.panw?.panos?.type == "TRAFFIC"' - set: field: event.kind value: alert - if: 'ctx?._temp_?.message_type == "THREAT"' + if: 'ctx?.panw?.panos?.type == "THREAT"' - append: field: event.category value: - security_threat - intrusion_detection - network - if: 'ctx?._temp_?.message_type == "THREAT"' + if: 'ctx?.panw?.panos?.type == "THREAT"' - append: field: event.type value: allowed @@ -217,89 +217,89 @@ processors: - set: field: event.action value: flow_started - if: 'ctx?._temp_?.message_subtype == "start"' + if: 'ctx?.panw?.panos?.sub_type == "start"' - append: field: event.type value: - start - connection - if: 'ctx?._temp_?.message_subtype == "start"' + if: 'ctx?.panw?.panos?.sub_type == "start"' - set: field: event.action value: flow_terminated - if: 'ctx?._temp_?.message_subtype == "end"' + if: 'ctx?.panw?.panos?.sub_type == "end"' - append: field: event.type value: - end - connection - if: 'ctx?._temp_?.message_subtype == "end"' + if: 'ctx?.panw?.panos?.sub_type == "end"' - set: field: event.action value: flow_dropped - if: 'ctx?._temp_?.message_subtype == "drop"' + if: 'ctx?.panw?.panos?.sub_type == "drop"' - append: field: event.type value: - denied - connection - if: 'ctx?._temp_?.message_subtype == "drop"' + if: 'ctx?.panw?.panos?.sub_type == "drop"' - set: field: event.action value: flow_denied - if: 'ctx?._temp_?.message_subtype == "deny"' + if: 'ctx?.panw?.panos?.sub_type == "deny"' - append: field: event.type value: - denied - connection - if: 'ctx?._temp_?.message_subtype == "deny"' + if: 'ctx?.panw?.panos?.sub_type == "deny"' # event.action for threat logs. - set: field: event.action value: data_match - if: 'ctx?._temp_?.message_subtype == "data"' + if: 'ctx?.panw?.panos?.sub_type == "data"' - set: field: event.action value: file_match - if: 'ctx?._temp_?.message_subtype == "file"' + if: 'ctx?.panw?.panos?.sub_type == "file"' - set: field: event.action value: flood_detected - if: 'ctx?._temp_?.message_subtype == "flood"' + if: 'ctx?.panw?.panos?.sub_type == "flood"' - set: field: event.action value: packet_attack - if: 'ctx?._temp_?.message_subtype == "packet"' + if: 'ctx?.panw?.panos?.sub_type == "packet"' - set: field: event.action value: scan_detected - if: 'ctx?._temp_?.message_subtype == "scan"' + if: 'ctx?.panw?.panos?.sub_type == "scan"' - set: field: event.action value: spyware_detected - if: 'ctx?._temp_?.message_subtype == "spyware"' + if: 'ctx?.panw?.panos?.sub_type == "spyware"' - set: field: event.action value: url_filtering - if: 'ctx?._temp_?.message_subtype == "url"' + if: 'ctx?.panw?.panos?.sub_type == "url"' - set: field: event.action value: virus_detected - if: 'ctx?._temp_?.message_subtype == "virus"' + if: 'ctx?.panw?.panos?.sub_type == "virus"' - set: field: event.action value: exploit_detected - if: 'ctx?._temp_?.message_subtype == "vulnerability"' + if: 'ctx?.panw?.panos?.sub_type == "vulnerability"' - set: field: event.action value: wildfire_verdict - if: 'ctx?._temp_?.message_subtype == "wildfire"' + if: 'ctx?.panw?.panos?.sub_type == "wildfire"' - set: field: event.action value: wildfire_virus_detected - if: 'ctx?._temp_?.message_subtype == "wildfire-virus"' + if: 'ctx?.panw?.panos?.sub_type == "wildfire-virus"' # Set numeric log.level from event.severity. diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json index 675df939dce..96530ab70f3 100644 --- a/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/pan_inc_other.log-expected.json @@ -13,6 +13,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -33,6 +35,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -53,6 +57,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -73,6 +79,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -93,6 +101,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -113,6 +123,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -133,6 +145,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -153,6 +167,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -173,6 +189,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -193,6 +211,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -213,6 +233,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -233,6 +255,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -253,6 +277,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -273,6 +299,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -293,6 +321,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "general", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -313,6 +343,8 @@ "observer.serial_number": "1606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "general", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -333,6 +365,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "general", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -353,6 +387,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -373,6 +409,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -393,6 +431,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -413,6 +453,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -433,6 +475,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -453,6 +497,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "general", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -473,6 +519,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -493,6 +541,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -513,6 +563,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "routing", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -533,6 +585,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -553,6 +607,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -573,6 +629,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -593,6 +651,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "0", + "panw.panos.type": "CONFIG", "service.type": "panw", "tags": [ "pan-os", @@ -613,6 +673,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "general", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -633,6 +695,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "ras", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os", @@ -653,6 +717,8 @@ "observer.serial_number": "01606001116", "observer.type": "firewall", "observer.vendor": "Palo Alto Networks", + "panw.panos.sub_type": "vpn", + "panw.panos.type": "SYSTEM", "service.type": "panw", "tags": [ "pan-os",