[Filebeat] Improve ASA/FTD Ingest Pipeline

CHANGELOG.next.asciidoc

@@ -376,6 +376,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697]
 - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
 - Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
+- Fix date parsing in GSuite/login fileset. {issue}24694[24694]
+- Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.  Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -21773,6 +21773,26 @@ type: keyword
 The total count of burst rate hits since the object was created or cleared
 
 
+type: keyword
+
+--
+
+*`cisco.asa.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.asa.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --
@@ -21991,6 +22011,26 @@ type: keyword
 The assigned DAP records
 
 
+type: keyword
+
+--
+
+*`cisco.ftd.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.ftd.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --

x-pack/filebeat/module/cisco/asa/_meta/fields.yml

@@ -175,3 +175,15 @@
     type: keyword
     description: >
       The total count of burst rate hits since the object was created or cleared
+
+  - name: termination_user
+    default_field: false
+    type: keyword
+    description: >
+      AAA name of user requesting termination
+
+  - name: webvpn.group_name
+    type: keyword
+    default_field: false
+    description: >
+      The WebVPN group name the user belongs to

x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json

@@ -120,6 +120,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 3,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -255,6 +257,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 1,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -587,9 +591,10 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 0,
+        "cisco.asa.icmp_type": 8,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302021",
-        "cisco.asa.source_username": "type",
         "destination.address": "192.168.2.2",
         "destination.ip": "192.168.2.2",
         "event.action": "flow-expiration",
@@ -771,6 +776,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 3,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -837,6 +844,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 2960892904 for out111:10.10.10.10/443 to fw111:192.168.2.2/55225 duration 0:00:00 bytes 0 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2021-05-05T20:29:32.000Z",
         "event.timezone": "-02:00",
@@ -1799,7 +1807,7 @@
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 4949,
-        "network.bytes": "0",
+        "network.bytes": 0,
         "network.iana_number": 6,
         "network.transport": "tcp",
         "observer.egress.interface.name": "fw111",
@@ -1854,7 +1862,7 @@
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 5142,
-        "network.bytes": "0",
+        "network.bytes": 0,
         "network.iana_number": 6,
         "network.transport": "tcp",
         "observer.egress.interface.name": "net",
@@ -2644,6 +2652,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302304: Teardown TCP state-bypass connection 2751765169 from server.deflan:1.2.3.4/54242 to server.deflan:2.3.4.5/9101 duration 1:00:02 bytes 245 Connection timeout",
+        "event.reason": "Connection timeout",
         "event.severity": 6,
         "event.start": "2021-04-27T05:12:21.000Z",
         "event.timezone": "-02:00",
@@ -3229,6 +3238,7 @@
     {
         "cisco.asa.assigned_ip": "192.168.50.5",
         "cisco.asa.message_id": "722051",
+        "cisco.asa.webvpn.group_name": "VPN5Policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3272,6 +3282,7 @@
     },
     {
         "cisco.asa.message_id": "716002",
+        "cisco.asa.webvpn.group_name": "another-policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3281,7 +3292,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-716002: Group another-policy User testuser IP 8.8.8.8 WebVPN session terminated: User Requested.",
-        "event.reason": "User Requested.",
+        "event.reason": "User Requested",
         "event.severity": 6,
         "event.timezone": "-02:00",
         "event.type": [
@@ -3323,6 +3334,7 @@
     },
     {
         "cisco.asa.message_id": "716002",
+        "cisco.asa.webvpn.group_name": "another-policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3332,7 +3344,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-716002: Group another-policy User alice IP 192.168.50.1 WebVPN session terminated: Idle timeout.",
-        "event.reason": "Idle timeout.",
+        "event.reason": "Idle timeout",
         "event.severity": 6,
         "event.timezone": "-02:00",
         "event.type": [

x-pack/filebeat/module/cisco/asa/test/asa-fix.log-expected.json

@@ -5,6 +5,7 @@
         "cisco.asa.message_id": "302016",
         "cisco.asa.source_interface": "Outside",
         "cisco.asa.source_username": "(LOCAL\\Elastic)",
+        "cisco.asa.termination_user": "zzzzzz",
         "destination.address": "10.233.123.123",
         "destination.ip": "10.233.123.123",
         "destination.port": 53,

x-pack/filebeat/module/cisco/asa/test/asa.log-expected.json

@@ -133,6 +133,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11749 for outside:100.66.211.242/80 to inside:172.31.98.44/1758 duration 0:01:07 bytes 38110 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -193,6 +194,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11748 for outside:100.66.211.242/80 to inside:172.31.98.44/1757 duration 0:01:07 bytes 44010 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -253,6 +255,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11745 for outside:100.66.185.90/80 to inside:172.31.98.44/1755 duration 0:01:07 bytes 7652 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -313,6 +316,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11744 for outside:100.66.185.90/80 to inside:172.31.98.44/1754 duration 0:01:07 bytes 7062 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -373,6 +377,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11742 for outside:100.66.160.197/80 to inside:172.31.98.44/1752 duration 0:01:08 bytes 5738 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -433,6 +438,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11738 for outside:100.66.205.14/80 to inside:172.31.98.44/1749 duration 0:01:08 bytes 4176 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -493,6 +499,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11739 for outside:100.66.124.33/80 to inside:172.31.98.44/1750 duration 0:01:08 bytes 1715 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -553,6 +560,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11731 for outside:100.66.35.9/80 to inside:172.31.98.44/1747 duration 0:01:09 bytes 45595 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -613,6 +621,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11723 for outside:100.66.211.242/80 to inside:172.31.98.44/1742 duration 0:01:09 bytes 27359 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -673,6 +682,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11715 for outside:100.66.218.21/80 to inside:172.31.98.44/1741 duration 0:01:09 bytes 4457 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -733,6 +743,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11711 for outside:100.66.198.27/80 to inside:172.31.98.44/1739 duration 0:01:09 bytes 26709 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -793,6 +804,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11712 for outside:100.66.198.27/80 to inside:172.31.98.44/1740 duration 0:01:09 bytes 22097 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -853,6 +865,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11708 for outside:100.66.202.211/80 to inside:172.31.98.44/1738 duration 0:01:10 bytes 2209 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:46.000Z",
         "event.timezone": "-02:00",
@@ -913,6 +926,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11746 for outside:100.66.124.15/80 to inside:172.31.98.44/1756 duration 0:01:07 bytes 10404 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -973,6 +987,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11706 for outside:100.66.124.15/80 to inside:172.31.98.44/1737 duration 0:01:10 bytes 123694 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:46.000Z",
         "event.timezone": "-02:00",
@@ -1033,6 +1048,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11702 for outside:100.66.209.247/80 to inside:172.31.98.44/1736 duration 0:01:11 bytes 35835 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:45.000Z",
         "event.timezone": "-02:00",
@@ -1093,6 +1109,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11753 for outside:100.66.35.162/80 to inside:172.31.98.44/1765 duration 0:00:30 bytes 0 SYN Timeout",
+        "event.reason": "SYN Timeout",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:26.000Z",
         "event.timezone": "-02:00",
@@ -2791,6 +2808,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11777 for outside:100.66.133.112/80 to inside:172.31.98.44/1453 duration 0:00:00 bytes 862 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",
@@ -3781,6 +3799,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11784 for outside:100.66.198.40/80 to inside:172.31.98.44/1457 duration 0:00:00 bytes 593 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",
@@ -4509,6 +4528,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11564 for outside:100.66.115.46/80 to inside:172.31.156.80/1382 duration 0:05:25 bytes 575 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:29:31.000Z",
         "event.timezone": "-02:00",
@@ -4569,6 +4589,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11797 for outside:100.66.19.254/80 to inside:172.31.156.80/1385 duration 0:00:00 bytes 5391 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",

x-pack/filebeat/module/cisco/asa/test/hostnames.log-expected.json

@@ -49,9 +49,10 @@
     },
     {
         "@timestamp": "2011-06-04T21:59:52.000-02:00",
+        "cisco.asa.icmp_code": 0,
+        "cisco.asa.icmp_type": 8,
         "cisco.asa.mapped_source_ip": "192.0.2.134",
         "cisco.asa.message_id": "302021",
-        "cisco.asa.source_username": "type",
         "destination.address": "192.0.2.15",
         "destination.ip": "192.0.2.15",
         "event.action": "flow-expiration",