Cherry-pick #23424 to 7.x: Sync fixes from Integration Package Testing

CHANGELOG.next.asciidoc

@@ -153,6 +153,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- CheckPoint Firewall module: Change event.severity JSON data type to a number because the field mapping is a `long`. {pull}23424[23424]
+- Cisco IOS: Change icmp.type/code and igmp.type JSON data types to strings because the fields mappings are `keyword`. {pull}23424[23424]
+- CrowdStrike Falcon: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Fortinet Firewall: Drop `fortinet.firewall.assignip` when the value is "N/A". {pull}23424[23424]
+- Juniper SRX: Change JSON field types to match the field mappings. {pull}23424[23424]
+- Suricata EVE: Convert `suricata.eve.flow_id` to string because the field is a keyword in the mapping. {pull}23424[23424]
+- Zeek DNS: Ignore failures in data type conversions. And change `dns.id` JSON field to a string to match its `keyword` mapping. {pull}23424[23424]
 - Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
 - cisco/asa fileset: Fix parsing of 302021 message code. {pull}14519[14519]
 - Fix filebeat azure dashboards, event category should be `Alert`. {pull}14668[14668]

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -152,25 +152,25 @@ processors:
             Map map = new HashMap();
             ctx.aws.cloudtrail.put("flattened", map);
           }
-        if (ctx.json.requestParameters != null) {
+        if (ctx.json?.requestParameters != null) {
           ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
           if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
             ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
           }
         }
-        if (ctx.json.responseElements != null) {
+        if (ctx.json?.responseElements != null) {
           ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
           if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
             ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
           }
         }
-        if (ctx.json.additionalEventData != null) {
+        if (ctx.json?.additionalEventData != null) {
           ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
           if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
             ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
           }
         }
-        if (ctx.json.serviceEventDetails != null) {
+        if (ctx.json?.serviceEventDetails != null) {
           ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
           if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
             ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);

x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml

@@ -309,9 +309,11 @@ processors:
     type: long
     ignore_failure: true
     ignore_missing: true
-- rename:
+- convert:
     field: checkpoint.severity
     target_field: event.severity
+    type: long
+    ignore_failure: true
     ignore_missing: true
 - rename:
     field: checkpoint.action
@@ -859,6 +861,7 @@ processors:
     - checkpoint.xlatedst
     - checkpoint.uid
     - checkpoint.time
+    - checkpoint.severity
     - syslog5424_ts
     - _temp_
     ignore_missing: true

x-pack/filebeat/module/cisco/ios/config/pipeline.js

@@ -166,9 +166,6 @@ var ciscoIOS = (function() {
             {from: "source.port", type: "long"},
             {from: "source.packets", type: "long"},
             {from: "source.packets", to: "network.packets", type: "long"},
-            {from: "icmp.type", type: "long"},
-            {from: "icmp.code", type: "long"},
-            {from: "igmp.type", type: "long"},
         ],
         ignore_missing: true,
     }).Run;

x-pack/filebeat/module/cisco/ios/pipeline_test.go

@@ -71,8 +71,8 @@ var testCases = []testCase{
 			"event.outcome":         "deny",
 			"event.severity":        int64(6),
 			"event.type":            []string{"connection", "firewall"},
-			"icmp.code":             int64(5),
-			"icmp.type":             int64(3),
+			"icmp.code":             "5",
+			"icmp.type":             "3",
 			"log.level":             "informational",
 			"log.original":          isdef.IsNonEmptyString,
 			"message":               "list 100 denied icmp 198.51.100.1 -> 198.51.100.2 (3/5), 1 packet",
@@ -117,7 +117,7 @@ var testCases = []testCase{
 			"event.outcome":         "deny",
 			"event.severity":        int64(6),
 			"event.type":            []string{"connection", "firewall"},
-			"igmp.type":             int64(20),
+			"igmp.type":             "20",
 			"log.level":             "informational",
 			"log.original":          isdef.IsNonEmptyString,
 			"message":               "list INBOUND-ON-AP denied igmp 198.51.100.1 -> 224.0.0.2 (20), 1 packet",

x-pack/filebeat/module/cisco/ios/test/cisco-ios-syslog.log-expected.json

@@ -66,7 +66,7 @@
             "firewall"
         ],
         "fileset.name": "ios",
-        "igmp.type": 20,
+        "igmp.type": "20",
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 140,
@@ -251,8 +251,8 @@
             "firewall"
         ],
         "fileset.name": "ios",
-        "icmp.code": 4,
-        "icmp.type": 3,
+        "icmp.code": "4",
+        "icmp.type": "3",
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 760,
@@ -1268,8 +1268,8 @@
             "firewall"
         ],
         "fileset.name": "ios",
-        "icmp.code": 3,
-        "icmp.type": 3,
+        "icmp.code": "3",
+        "icmp.type": "3",
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 4125,

x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml

@@ -71,7 +71,7 @@ processors:
   # Parse the date included in FTD logs
   #
   - date:
-      if: "ctx.event.timezone == null"
+      if: "ctx.event?.timezone == null && ctx._temp_?.raw_date != null"
       field: "_temp_.raw_date"
       target_field: "@timestamp"
       formats:
@@ -103,7 +103,7 @@ processors:
           },
         ]
   - date:
-      if: "ctx.event.timezone != null"
+      if: "ctx.event?.timezone != null && ctx._temp_?.raw_date != null"
       timezone: "{{ event.timezone }}"
       field: "_temp_.raw_date"
       target_field: "@timestamp"

x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js

@@ -12,6 +12,10 @@ var crowdstrikeFalconProcessor = (function () {
 
     function convertToMSEpoch(evt, field) {
         var timestamp = evt.Get(field);
+        if (timestamp == 0) {
+            evt.Delete(field)
+            return
+        }
         if (timestamp) {
             if (timestamp < 100000000000) { // check if we have a seconds timestamp, this is roughly 1973 in MS
                 evt.Put(field, timestamp * 1000);
@@ -103,7 +107,8 @@ var crowdstrikeFalconProcessor = (function () {
                         type: "ip"
                     }, {
                         from: "crowdstrike.event.ProcessId",
-                        to: "process.pid"
+                        to: "process.pid",
+                        type: "long"
                     }, {
                         from: "crowdstrike.event.ParentImageFileName",
                         to: "process.parent.executable"
@@ -284,6 +289,7 @@ var crowdstrikeFalconProcessor = (function () {
                     }, {
                         from: "crowdstrike.event.PID",
                         to: "process.pid",
+                        type: "long"
                     },
                     {
                         from: "crowdstrike.event.RuleId",
@@ -421,6 +427,44 @@ var crowdstrikeFalconProcessor = (function () {
             ignore_missing: false,
             fail_on_error: true
         })
+        .Convert({
+            fields: [
+                {
+                    from: "crowdstrike.event.LateralMovement",
+                    type: "long",
+                },
+                {
+                    from: "crowdstrike.event.LocalPort",
+                    type: "long",
+                },
+                {
+                    from: "crowdstrike.event.MatchCount",
+                    type: "long",
+                },
+                {
+                    from: "crowdstrike.event.MatchCountSinceLastReport",
+                    type: "long",
+                },
+                {
+                    from: "crowdstrike.event.PID",
+                    type: "long",
+                },
+                {
+                    from: "crowdstrike.event.RemotePort",
+                    type: "long",
+                },
+                {
+                    from: "source.port",
+                    type: "long",
+                },
+                {
+                    from: "destination.port",
+                    type: "long",
+                }
+            ],
+            ignore_missing: true,
+            fail_on_error: false
+        })
         .Build()
         .Run
 })();

x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json

@@ -29,7 +29,6 @@
         "crowdstrike.event.PatternDispositionFlags.Rooting": false,
         "crowdstrike.event.PatternDispositionFlags.SensorOnly": false,
         "crowdstrike.event.PatternDispositionValue": 16,
-        "crowdstrike.event.ProcessEndTime": 0,
         "crowdstrike.event.ProcessId": 38684386611,
         "crowdstrike.event.ProcessStartTime": "2018-09-13T13:45:39.000Z",
         "crowdstrike.event.SHA256String": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a",

x-pack/filebeat/module/crowdstrike/falcon/test/falcon-sample.log-expected.json

@@ -10,16 +10,16 @@
         "crowdstrike.event.HostName": "TESTDEVICE01",
         "crowdstrike.event.Ipv": "ipv4",
         "crowdstrike.event.LocalAddress": "10.37.60.194",
-        "crowdstrike.event.LocalPort": "445",
+        "crowdstrike.event.LocalPort": 445,
         "crowdstrike.event.MatchCount": 1,
         "crowdstrike.event.MatchCountSinceLastReport": 1,
         "crowdstrike.event.NetworkProfile": "2",
-        "crowdstrike.event.PID": "206158879910",
+        "crowdstrike.event.PID": 206158879910,
         "crowdstrike.event.PolicyID": "74e7f1552a3a4d90a6d65578642c8584",
         "crowdstrike.event.PolicyName": "PROD-FW-Workstations-General",
         "crowdstrike.event.Protocol": "6",
         "crowdstrike.event.RemoteAddress": "10.37.60.21",
-        "crowdstrike.event.RemotePort": "54952",
+        "crowdstrike.event.RemotePort": 54952,
         "crowdstrike.event.RuleAction": "2",
         "crowdstrike.event.RuleFamilyID": "fec73e96a1bf4481be582c3f89b234fa",
         "crowdstrike.event.RuleGroupName": "SMB Rules",
@@ -32,7 +32,7 @@
         "crowdstrike.metadata.offset": 70689,
         "crowdstrike.metadata.version": "1.0",
         "destination.ip": "10.37.60.194",
-        "destination.port": "445",
+        "destination.port": 445,
         "event.action": "firewall_match_event",
         "event.category": [
             "network"
@@ -58,7 +58,7 @@
         "message": "Firewall Rule 'Inbound SMB Block & Log Private' triggered",
         "network.direction": "ingress",
         "network.type": "ipv4",
-        "process.pid": "206158879910",
+        "process.pid": 206158879910,
         "related.ip": [
             "10.37.60.21",
             "10.37.60.194"
@@ -70,7 +70,7 @@
         "rule.ruleset": "SMB Rules",
         "service.type": "crowdstrike",
         "source.ip": "10.37.60.21",
-        "source.port": "54952",
+        "source.port": 54952,
         "tags": [
             "forwarded"
         ]

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -80,26 +80,26 @@ processors:
     formats:
     - UNIX_MS
     timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
     timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
-    if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - script:
     lang: painless
     source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"
@@ -134,6 +134,9 @@ processors:
     field: fortinet.firewall.level
     target_field: log.level
     ignore_missing: true
+- remove:
+    field: fortinet.firewall.assignip
+    if: "ctx.fortinet?.firewall?.assignip == 'N/A'"
 - remove:
     field: fortinet.firewall.dstip
     if: "ctx.fortinet?.firewall?.dstip == 'N/A'"
@@ -222,16 +225,18 @@ processors:
       )
 - remove:
     field:
-    - _temp
-    - message
-    - syslog5424_sd
-    - syslog5424_pri
-    - fortinet.firewall.tz
-    - fortinet.firewall.date
-    - fortinet.firewall.eventtime
-    - fortinet.firewall.time
-    - fortinet.firewall.duration
-    - host
+      - _temp.time
+      - _temp
+      - message
+      - syslog5424_sd
+      - syslog5424_pri
+      - fortinet.firewall.tz
+      - fortinet.firewall.date
+      - fortinet.firewall.devid
+      - fortinet.firewall.eventtime
+      - fortinet.firewall.time
+      - fortinet.firewall.duration
+      - host
     ignore_missing: true
 - pipeline:
     name: '{< IngestPipeline "event" >}'
@@ -242,6 +247,18 @@ processors:
 - pipeline:
     name: '{< IngestPipeline "utm" >}'
     if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
+- convert:
+    field: fortinet.firewall.quotamax
+    type: long
+    ignore_missing: true
+- convert:
+    field: fortinet.firewall.quotaused
+    type: long
+    ignore_missing: true
+- convert:
+    field: fortinet.firewall.size
+    type: long
+    ignore_missing: true
 on_failure:
 - set:
     field: error.message

x-pack/filebeat/module/juniper/srx/ingest/flow.yml

@@ -13,11 +13,12 @@ processors:
 - append:
     field: event.category
     value: network
-- rename:
+- convert:
     field: juniper.srx.application_risk
+    type: float
     target_field: event.risk_score
     ignore_missing: true
-    if: "ctx.juniper?.srx?.application_risk != null"
+    ignore_failure: true
 - append:
     field: event.type
     value:
@@ -344,6 +345,7 @@ processors:
 #############
 - remove:
     field:
+    - juniper.srx.application_risk
     - juniper.srx.destination_port
     - juniper.srx.nat_destination_port
     - juniper.srx.bytes_from_client