Cherry-pick #25421 to 7.13: [Filebeat] Allow m365 defender to receive…

… incidents with no alerts (#25486) * [Filebeat] Allow m365 defender to receive incidents with no alerts (#25421) * allowing incidents with no alerts to be parsed, updating some typos and making the pipeline safer in terms of null def references * adding changelog entry * updating local test files * reverting changes to json decoding, back to beat (cherry picked from commit 733b655) * Update CHANGELOG.next.asciidoc
elastic · May 3, 2021 · a744117 · a744117
1 parent 933e906
commit a744117
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 27 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -165,6 +165,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887]
 - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764]
 - system/socket: Fixed tracking of long-running connections. {pull}19033[19033]
+- Fix o365 module config when client_secret contains special characters. {issue}25058[25058]
 
 *Filebeat*
 

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml
@@ -10,6 +10,11 @@ processors:
       - json.comments
       - host
     ignore_missing: true
+- remove:
+    field:
+      - json.alerts
+    ignore_missing: true
+    if: 'ctx?.json?.alerts == null || ctx?.json?.alerts instanceof List'
 
 #########################
 ## ECS General Mapping ##
@@ -45,11 +50,16 @@ processors:
 - set:
     field: '@timestamp'
     value: '{{json.lastUpdateTime}}'
-    if: ctx.json?.lastUpdateTime != null
+    if: ctx?.json?.lastUpdateTime != null
 - rename:
     field: json.alerts.title
     target_field: message
     ignore_missing: true
+- rename:
+    field: json.incidentName
+    target_field: message
+    ignore_missing: true
+    if: ctx?.message == null
 
 #######################
 ## ECS Event Mapping ##
@@ -144,7 +154,7 @@ processors:
 - set:
     field: threat.framework
     value: MITRE ATT&CK
-    if: ctx.json?.alerts?.category != null
+    if: ctx?.json?.alerts?.category != null
 - rename:
     field: json.alerts.category
     target_field: threat.technique.name
@@ -153,7 +163,7 @@ processors:
     field: json.alerts.description
     target_field: rule.description
     ignore_missing: true
-    if: ctx.json?.alerts?.description.length() < 1020
+    if: ctx?.json?.alerts != null && ctx.json?.alerts?.description.length() < 1020
 
 ######################
 ## ECS File Mapping ##
@@ -220,8 +230,11 @@ processors:
     field: json.alerts.entities.url
     target_field: url.full
     ignore_missing: true
-    if: ctx?.json?.entities?.url != null
-
+    if: ctx?.json?.alerts?.entities?.url != null
+- uri_parts:
+    field: url.full
+    ignore_failure: true
+    if: ctx?.url?.full != null
 ######################
 ## ECS User Mapping ##
 ######################
@@ -261,23 +274,23 @@ processors:
 - append:
     field: related.ip
     value: '{{json.alerts.entities.ipAddress}}'
-    if: ctx.json?.entities?.ipAddress != null
+    if: ctx?.json?.alerts?.entities?.ipAddress != null
 - append:
     field: related.user
     value: '{{user.name}}'
-    if: ctx.user?.name != null
+    if: ctx?.user?.name != null
 - append:
     field: related.hash
     value: '{{file.hash.sha1}}'
-    if: ctx.file?.hash?.sha1 != null
+    if: ctx?.file?.hash?.sha1 != null
 - append:
     field: related.hash
     value: '{{file.hash.sha256}}'
-    if: ctx.file?.hash?.sha256 != null
+    if: ctx?.file?.hash?.sha256 != null
 - append:
     field: related.hosts
     value: '{{host.hostname}}'
-    if: ctx.host?.hostname != null
+    if: ctx?.host?.hostname != null
 
 #############
 ## Cleanup ##
@@ -293,15 +306,15 @@ processors:
 - remove:
     field: json.alerts.mitreTechniques
     ignore_missing: true
-    if: ctx?.json?.alerts?.mitreTechniques.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.mitreTechniques.isEmpty()'
 - remove:
     field: json.alerts.devices
     ignore_missing: true
-    if: ctx?.json?.alerts?.devices.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.devices.isEmpty()'
 - remove:
     field: json.tags
     ignore_missing: true
-    if: ctx?.json?.tags.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.tags.isEmpty()'
 - remove:
     ignore_missing: true
     field:

diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log
@@ -0,0 +1 @@
+{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}
diff --git a/...eat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json b/...eat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json
@@ -0,0 +1,30 @@
+[
+    {
+        "@timestamp": "2021-04-12T11:18:30.4033333Z",
+        "cloud.provider": "azure",
+        "event.category": [
+            "host"
+        ],
+        "event.dataset": "microsoft.m365_defender",
+        "event.kind": "alert",
+        "event.module": "microsoft",
+        "event.timezone": "UTC",
+        "fileset.name": "m365_defender",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "Impossible travel activity involving one user",
+        "microsoft.m365_defender.classification": "Unknown",
+        "microsoft.m365_defender.determination": "NotAvailable",
+        "microsoft.m365_defender.incidentId": "1111",
+        "microsoft.m365_defender.redirectIncidentId": 1107,
+        "microsoft.m365_defender.status": "Redirected",
+        "microsoft.m365_defender.tags": [],
+        "observer.product": "365 Defender",
+        "observer.vendor": "Microsoft",
+        "service.type": "microsoft",
+        "tags": [
+            "m365-defender",
+            "forwarded"
+        ]
+    }
+]
diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log
@@ -5,5 +5,5 @@
 {"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","determination":null,"serviceSource":"MicrosoftDefenderATP","severity":"Low","alertId":"da637291086161511365_-2075772905","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:31:04.1092404Z","processId":6720},"mitreTechniques":[],"title":"Suspicious 'AccessibilityEscalation' behavior was detected","category":"SuspiciousActivity","devices":[{"aadDeviceId":null,"mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osProcessor":"x64","riskScore":"High","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"version":"Other","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","status":"Resolved","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","resolvedTime":"2020-09-23T19:44:36.1092821Z","threatFamilyName":null,"actorName":null},"determination":"NotAvailable","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z"}
 {"determination":"NotAvailable","severity":"Low","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"alerts":{"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","determination":null,"entities":{"accountName":"","entityType":"User"},"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","serviceSource":"MicrosoftDefenderATP","status":"Resolved","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","devices":[{"aadDeviceId":null,"healthStatus":"Inactive","osPlatform":"Other","osProcessor":"x64","riskScore":"High","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"rbacGroupId":0,"rbacGroupName":null,"version":"Other"}],"mitreTechniques":[],"severity":"Low","threatFamilyName":null,"creationTime":"2020-06-30T10:10:16.1355657Z","detectionSource":"WindowsDefenderAv","incidentId":12,"alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","category":"SuspiciousActivity","investigationId":null,"resolvedTime":"2020-09-23T19:44:36.1092821Z"},"assignedTo":"elastic@elasticuser.com","status":"Resolved","tags":[]}
 {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"}
-{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"}
 {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"}
+{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"}
diff --git a/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/.../filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json
@@ -521,17 +521,15 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
-        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "host.user.name": "brent@elasticbv.onmicrosoft.com",
         "input.type": "log",
         "log.offset": 14764,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.accountName": "brent",
-        "microsoft.m365_defender.alerts.entities.entityType": "User",
+        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
+        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -546,8 +544,8 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
-        "related.user": [
-            "brent@elasticbv.onmicrosoft.com"
+        "related.ip": [
+            "73.172.171.53"
         ],
         "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
@@ -556,9 +554,7 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity",
-        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "user.name": "brent@elasticbv.onmicrosoft.com"
+        "threat.technique.name": "SuspiciousActivity"
     },
     {
         "@timestamp": "2020-09-23T19:32:05.8366667Z",
@@ -578,15 +574,17 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
+        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "host.user.name": "brent@elasticbv.onmicrosoft.com",
         "input.type": "log",
-        "log.offset": 16091,
+        "log.offset": 15990,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
-        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
+        "microsoft.m365_defender.alerts.entities.accountName": "brent",
+        "microsoft.m365_defender.alerts.entities.entityType": "User",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -601,13 +599,18 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
+        "related.user": [
+            "brent@elasticbv.onmicrosoft.com"
+        ],
         "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
         "tags": [
             "m365-defender",
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity"
+        "threat.technique.name": "SuspiciousActivity",
+        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "user.name": "brent@elasticbv.onmicrosoft.com"
     }
 ]