From cf2c984ad26000c4a950b06cbbae6b3465db3ab4 Mon Sep 17 00:00:00 2001 From: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Date: Mon, 11 Apr 2022 14:15:12 -0500 Subject: [PATCH] [Filebeat] fix m365_defender pipeline bug (#31227) - remove `alerts.entities` when it is an empty list. Prevents errors where `alerts.entities` is assumed to be a Map. Closes #31223 (cherry picked from commit 60fcf9d6f210258ab8a1b56039fa4eb65f1aeb54) --- CHANGELOG.next.asciidoc | 4 +- .../m365_defender/ingest/pipeline.yml | 3 + .../test/m365_defender-test.ndjson.log | 1 + ...365_defender-test.ndjson.log-expected.json | 88 ++++++++++++++++++- 4 files changed, 90 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c8cfafd397b..04a5aa8c90d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,15 +54,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - auditd: Prevent mapping explosion when truncated EXECVE records are ingested. {pull}30382[30382] - elasticsearch: fix duplicate ingest when using a common appender configuration {issue}30428[30428] {pull}30440[30440] - Prevent logic race on clearing data during request in httpjson. {pull}30730[30730] - -*Filebeat* - - Recover CEF extensions from messages with invalid/incomplete headers. {issue}30757[30757] {pull}30938[30938] - Fix panic in filestream input when `copy_truncate` log rotation strategy is used {issue}29024[29024] {pull}31041[31041] - Fix Azure signinlogs authentication_requirement_policies field type and several missing fields. {pull}31062[31062] - Fix Azure activitylogs identity field type and several missing fields. {pull}31170[31170] - Cyberark PAS: Fix error ingesting events with a single entry in the CAProperties field. {pull}31094[31094] - checkpoint: Fix ingest error when a message contains trailing spaces {pull}31197[31197] +- m365_defender: Fix processing when alerts.entities is an empty list. {issue}31223[31223] {pull}31227[31227] *Heartbeat* diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index c1edf50171b..ce339b3e785 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -168,6 +168,9 @@ processors: ###################### ## ECS File Mapping ## ###################### +- remove: + field: json.alerts.entities + if: ctx?.json?.alerts?.entities != null && ctx.json.alerts.entities instanceof List && ctx.json.alerts.entities.isEmpty() - rename: field: json.alerts.entities.fileName target_field: file.name diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log index b07d3846514..3d149cdda1e 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log @@ -7,3 +7,4 @@ {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"} {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"} {"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"} +{"status":"Resolved","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low","alerts":{"creationTime":"2020-06-30T09:32:31.4579225Z","detectionSource":"WindowsDefenderAv","firstActivity":"2020-06-30T09:31:22.5729558Z","incidentId":12,"serviceSource":"MicrosoftDefenderATP","actorName":null,"alertId":"da637291063515066999_-2102938302","determination":null,"lastActivity":"2020-06-30T09:46:15.0876676Z","assignedTo":"Automation","devices":[{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer4","healthStatus":"Inactive","osPlatform":"Other"},{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer5","healthStatus":"Inactive","osPlatform":"Other"}],"investigationId":9,"threatFamilyName":null,"title":"'Mountsi' malware was detected","category":"Malware","classification":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","entities":[],"investigationState":"Benign","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","severity":"Informational","status":"Resolved"},"assignedTo":"elastic@elasticuser.com","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[]} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json index e5e7070d1ae..3e57d460d68 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json @@ -31,7 +31,7 @@ "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", "microsoft.m365_defender.alerts.devices": [ { - "deviceDnsName": "TestServer4", + "deviceDnsName": "TestServer5", "firstSeen": "2020-06-30T08:55:08.8320449Z", "healthStatus": "Inactive", "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", @@ -43,7 +43,7 @@ "version": "Other" }, { - "deviceDnsName": "TestServer5", + "deviceDnsName": "TestServer4", "firstSeen": "2020-06-30T08:55:08.8320449Z", "healthStatus": "Inactive", "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", @@ -644,5 +644,87 @@ "threat.technique.name": "SuspiciousActivity", "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", "user.name": "brent@elasticbv.onmicrosoft.com" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "Malware", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 892514711800, + "event.end": "2020-06-30T09:46:15.0876676Z", + "event.id": "da637291063515066999_-2102938302", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T09:31:22.5729558Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 17627, + "message": "'Mountsi' malware was detected", + "microsoft.m365_defender.alerts.assignedTo": "Automation", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T09:32:31.4579225Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "TestServer4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + }, + { + "deviceDnsName": "TestServer5", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationId": 9, + "microsoft.m365_defender.alerts.investigationState": "Benign", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z", + "microsoft.m365_defender.alerts.severity": "Informational", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "related.hosts": [ + "TestServer4", + "TestServer5" + ], + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", + "service.type": "microsoft", + "tags": [ + "forwarded", + "m365-defender" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "Malware" } -] +] \ No newline at end of file