[Filebeat] fix m365_defender pipeline bug

- remove `alerts.entities` when it is an empty list. Prevents errors where `alerts.entities` is assumed to be a Map. Closes 31223
elastic · Apr 11, 2022 · f36e586 · f36e586
1 parent 19718ba
commit f36e586
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -56,15 +56,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Prevent logic race on clearing data during request in httpjson. {pull}30730[30730]
 - Do not emit error log when filestream reader reaches EOF and `close.reader.on_eof` is enabled. {pull}31109[31109]
 - Prevents filestream inputs from being stuck while being created. {pull}31240[31240]
-
-*Filebeat*
-
 - Recover CEF extensions from messages with invalid/incomplete headers. {issue}30757[30757] {pull}30938[30938]
 - Fix panic in filestream input when `copy_truncate` log rotation strategy is used {issue}29024[29024] {pull}31041[31041]
 - Fix Azure signinlogs authentication_requirement_policies field type and several missing fields. {pull}31062[31062]
 - Cyberark PAS: Fix error ingesting events with a single entry in the CAProperties field. {pull}31094[31094]
 - Fix Azure activitylogs identity field type and several missing fields. {pull}31170[31170]
 - checkpoint: Fix ingest error when a message contains trailing spaces {pull}31197[31197]
+- m365_defender: Fix processing when alerts.entities is an empty list. {issue}31223[31223] {pull}31227[31227]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml
@@ -168,6 +168,9 @@ processors:
 ######################
 ## ECS File Mapping ##
 ######################
+- remove:
+    field: json.alerts.entities
+    if: ctx?.json?.alerts?.entities != null && ctx.json.alerts.entities instanceof List && ctx.json.alerts.entities.isEmpty()
 - rename:
     field: json.alerts.entities.fileName
     target_field: file.name

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log
@@ -7,3 +7,4 @@
 {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"}
 {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"}
 {"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"}
+{"status":"Resolved","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low","alerts":{"creationTime":"2020-06-30T09:32:31.4579225Z","detectionSource":"WindowsDefenderAv","firstActivity":"2020-06-30T09:31:22.5729558Z","incidentId":12,"serviceSource":"MicrosoftDefenderATP","actorName":null,"alertId":"da637291063515066999_-2102938302","determination":null,"lastActivity":"2020-06-30T09:46:15.0876676Z","assignedTo":"Automation","devices":[{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer4","healthStatus":"Inactive","osPlatform":"Other"},{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer5","healthStatus":"Inactive","osPlatform":"Other"}],"investigationId":9,"threatFamilyName":null,"title":"'Mountsi' malware was detected","category":"Malware","classification":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","entities":[],"investigationState":"Benign","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","severity":"Informational","status":"Resolved"},"assignedTo":"elastic@elasticuser.com","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[]}
diff --git a/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json
@@ -31,7 +31,7 @@
         "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv",
         "microsoft.m365_defender.alerts.devices": [
             {
-                "deviceDnsName": "TestServer4",
+                "deviceDnsName": "TestServer5",
                 "firstSeen": "2020-06-30T08:55:08.8320449Z",
                 "healthStatus": "Inactive",
                 "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
@@ -43,7 +43,7 @@
                 "version": "Other"
             },
             {
-                "deviceDnsName": "TestServer5",
+                "deviceDnsName": "TestServer4",
                 "firstSeen": "2020-06-30T08:55:08.8320449Z",
                 "healthStatus": "Inactive",
                 "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
@@ -644,5 +644,87 @@
         "threat.technique.name": "SuspiciousActivity",
         "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
         "user.name": "brent@elasticbv.onmicrosoft.com"
+    },
+    {
+        "@timestamp": "2020-09-23T19:44:36.29Z",
+        "cloud.provider": "azure",
+        "event.action": "Malware",
+        "event.category": [
+            "host"
+        ],
+        "event.dataset": "microsoft.m365_defender",
+        "event.duration": 892514711800,
+        "event.end": "2020-06-30T09:46:15.0876676Z",
+        "event.id": "da637291063515066999_-2102938302",
+        "event.kind": "alert",
+        "event.module": "microsoft",
+        "event.provider": "MicrosoftDefenderATP",
+        "event.severity": 2,
+        "event.start": "2020-06-30T09:31:22.5729558Z",
+        "event.timezone": "UTC",
+        "event.type": [
+            "end"
+        ],
+        "fileset.name": "m365_defender",
+        "input.type": "log",
+        "log.offset": 17627,
+        "message": "'Mountsi' malware was detected",
+        "microsoft.m365_defender.alerts.assignedTo": "Automation",
+        "microsoft.m365_defender.alerts.creationTime": "2020-06-30T09:32:31.4579225Z",
+        "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv",
+        "microsoft.m365_defender.alerts.devices": [
+            {
+                "deviceDnsName": "TestServer4",
+                "firstSeen": "2020-06-30T08:55:08.8320449Z",
+                "healthStatus": "Inactive",
+                "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
+                "osBuild": 17763,
+                "osPlatform": "Other",
+                "osProcessor": "x64",
+                "rbacGroupId": 0,
+                "riskScore": "High",
+                "version": "Other"
+            },
+            {
+                "deviceDnsName": "TestServer5",
+                "firstSeen": "2020-06-30T08:55:08.8320449Z",
+                "healthStatus": "Inactive",
+                "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d",
+                "osBuild": 17763,
+                "osPlatform": "Other",
+                "osProcessor": "x64",
+                "rbacGroupId": 0,
+                "riskScore": "High",
+                "version": "Other"
+            }
+        ],
+        "microsoft.m365_defender.alerts.incidentId": "12",
+        "microsoft.m365_defender.alerts.investigationId": 9,
+        "microsoft.m365_defender.alerts.investigationState": "Benign",
+        "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z",
+        "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z",
+        "microsoft.m365_defender.alerts.severity": "Informational",
+        "microsoft.m365_defender.alerts.status": "Resolved",
+        "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com",
+        "microsoft.m365_defender.classification": "Unknown",
+        "microsoft.m365_defender.determination": "NotAvailable",
+        "microsoft.m365_defender.incidentId": "12",
+        "microsoft.m365_defender.incidentName": "12",
+        "microsoft.m365_defender.status": "Resolved",
+        "observer.name": "MicrosoftDefenderATP",
+        "observer.product": "365 Defender",
+        "observer.vendor": "Microsoft",
+        "related.hosts": [
+            "TestServer4",
+            "TestServer5"
+        ],
+        "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.",
+        "service.type": "microsoft",
+        "tags": [
+            "forwarded",
+            "m365-defender"
+        ],
+        "threat.framework": "MITRE ATT&CK",
+        "threat.technique.name": "Malware"
     }
-]
+]