Cherry-pick #25421 to 7.13: [Filebeat] Allow m365 defender to receive incidents with no alerts

CHANGELOG.next.asciidoc

@@ -165,6 +165,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887]
 - system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764]
 - system/socket: Fixed tracking of long-running connections. {pull}19033[19033]
+- Fix o365 module config when client_secret contains special characters. {issue}25058[25058]
 
 *Filebeat*
 

x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml

@@ -10,6 +10,11 @@ processors:
       - json.comments
       - host
     ignore_missing: true
+- remove:
+    field:
+      - json.alerts
+    ignore_missing: true
+    if: 'ctx?.json?.alerts == null || ctx?.json?.alerts instanceof List'
 
 #########################
 ## ECS General Mapping ##
@@ -45,11 +50,16 @@ processors:
 - set:
     field: '@timestamp'
     value: '{{json.lastUpdateTime}}'
-    if: ctx.json?.lastUpdateTime != null
+    if: ctx?.json?.lastUpdateTime != null
 - rename:
     field: json.alerts.title
     target_field: message
     ignore_missing: true
+- rename:
+    field: json.incidentName
+    target_field: message
+    ignore_missing: true
+    if: ctx?.message == null
 
 #######################
 ## ECS Event Mapping ##
@@ -144,7 +154,7 @@ processors:
 - set:
     field: threat.framework
     value: MITRE ATT&CK
-    if: ctx.json?.alerts?.category != null
+    if: ctx?.json?.alerts?.category != null
 - rename:
     field: json.alerts.category
     target_field: threat.technique.name
@@ -153,7 +163,7 @@ processors:
     field: json.alerts.description
     target_field: rule.description
     ignore_missing: true
-    if: ctx.json?.alerts?.description.length() < 1020
+    if: ctx?.json?.alerts != null && ctx.json?.alerts?.description.length() < 1020
 
 ######################
 ## ECS File Mapping ##
@@ -220,8 +230,11 @@ processors:
     field: json.alerts.entities.url
     target_field: url.full
     ignore_missing: true
-    if: ctx?.json?.entities?.url != null
-
+    if: ctx?.json?.alerts?.entities?.url != null
+- uri_parts:
+    field: url.full
+    ignore_failure: true
+    if: ctx?.url?.full != null
 ######################
 ## ECS User Mapping ##
 ######################
@@ -261,23 +274,23 @@ processors:
 - append:
     field: related.ip
     value: '{{json.alerts.entities.ipAddress}}'
-    if: ctx.json?.entities?.ipAddress != null
+    if: ctx?.json?.alerts?.entities?.ipAddress != null
 - append:
     field: related.user
     value: '{{user.name}}'
-    if: ctx.user?.name != null
+    if: ctx?.user?.name != null
 - append:
     field: related.hash
     value: '{{file.hash.sha1}}'
-    if: ctx.file?.hash?.sha1 != null
+    if: ctx?.file?.hash?.sha1 != null
 - append:
     field: related.hash
     value: '{{file.hash.sha256}}'
-    if: ctx.file?.hash?.sha256 != null
+    if: ctx?.file?.hash?.sha256 != null
 - append:
     field: related.hosts
     value: '{{host.hostname}}'
-    if: ctx.host?.hostname != null
+    if: ctx?.host?.hostname != null
 
 #############
 ## Cleanup ##
@@ -293,15 +306,15 @@ processors:
 - remove:
     field: json.alerts.mitreTechniques
     ignore_missing: true
-    if: ctx?.json?.alerts?.mitreTechniques.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.mitreTechniques.isEmpty()'
 - remove:
     field: json.alerts.devices
     ignore_missing: true
-    if: ctx?.json?.alerts?.devices.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.alerts?.devices.isEmpty()'
 - remove:
     field: json.tags
     ignore_missing: true
-    if: ctx?.json?.tags.isEmpty()
+    if: 'ctx?.json?.alerts != null && ctx?.json?.tags.isEmpty()'
 - remove:
     ignore_missing: true
     field:

x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log

@@ -0,0 +1 @@
+{"incidentId":1111,"redirectIncidentId":1107,"incidentName":"Impossible travel activity involving one user","createdTime":"2021-04-12T11:18:28.86Z","lastUpdateTime":"2021-04-12T11:18:30.4033333Z","assignedTo":null,"classification":"Unknown","determination":"NotAvailable","status":"Redirected","severity":"UnSpecified","tags":[],"comments":[],"alerts":[]}

x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test-empty.ndjson.log-expected.json

@@ -0,0 +1,30 @@
+[
+    {
+        "@timestamp": "2021-04-12T11:18:30.4033333Z",
+        "cloud.provider": "azure",
+        "event.category": [
+            "host"
+        ],
+        "event.dataset": "microsoft.m365_defender",
+        "event.kind": "alert",
+        "event.module": "microsoft",
+        "event.timezone": "UTC",
+        "fileset.name": "m365_defender",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "Impossible travel activity involving one user",
+        "microsoft.m365_defender.classification": "Unknown",
+        "microsoft.m365_defender.determination": "NotAvailable",
+        "microsoft.m365_defender.incidentId": "1111",
+        "microsoft.m365_defender.redirectIncidentId": 1107,
+        "microsoft.m365_defender.status": "Redirected",
+        "microsoft.m365_defender.tags": [],
+        "observer.product": "365 Defender",
+        "observer.vendor": "Microsoft",
+        "service.type": "microsoft",
+        "tags": [
+            "m365-defender",
+            "forwarded"
+        ]
+    }
+]

x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log

@@ -5,5 +5,5 @@
 {"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","determination":null,"serviceSource":"MicrosoftDefenderATP","severity":"Low","alertId":"da637291086161511365_-2075772905","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:31:04.1092404Z","processId":6720},"mitreTechniques":[],"title":"Suspicious 'AccessibilityEscalation' behavior was detected","category":"SuspiciousActivity","devices":[{"aadDeviceId":null,"mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osProcessor":"x64","riskScore":"High","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"version":"Other","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","status":"Resolved","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","resolvedTime":"2020-09-23T19:44:36.1092821Z","threatFamilyName":null,"actorName":null},"determination":"NotAvailable","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z"}
 {"determination":"NotAvailable","severity":"Low","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"alerts":{"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","determination":null,"entities":{"accountName":"","entityType":"User"},"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","serviceSource":"MicrosoftDefenderATP","status":"Resolved","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","devices":[{"aadDeviceId":null,"healthStatus":"Inactive","osPlatform":"Other","osProcessor":"x64","riskScore":"High","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"rbacGroupId":0,"rbacGroupName":null,"version":"Other"}],"mitreTechniques":[],"severity":"Low","threatFamilyName":null,"creationTime":"2020-06-30T10:10:16.1355657Z","detectionSource":"WindowsDefenderAv","incidentId":12,"alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","category":"SuspiciousActivity","investigationId":null,"resolvedTime":"2020-09-23T19:44:36.1092821Z"},"assignedTo":"elastic@elasticuser.com","status":"Resolved","tags":[]}
 {"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"}
-{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"}
 {"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"}
+{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"}

x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json

@@ -521,17 +521,15 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
-        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "host.user.name": "brent@elasticbv.onmicrosoft.com",
         "input.type": "log",
         "log.offset": 14764,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.accountName": "brent",
-        "microsoft.m365_defender.alerts.entities.entityType": "User",
+        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
+        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -546,8 +544,8 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
-        "related.user": [
-            "brent@elasticbv.onmicrosoft.com"
+        "related.ip": [
+            "73.172.171.53"
         ],
         "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
@@ -556,9 +554,7 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity",
-        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
-        "user.name": "brent@elasticbv.onmicrosoft.com"
+        "threat.technique.name": "SuspiciousActivity"
     },
     {
         "@timestamp": "2020-09-23T19:32:05.8366667Z",
@@ -578,15 +574,17 @@
         "event.start": "2020-07-27T15:47:22.088Z",
         "event.timezone": "UTC",
         "fileset.name": "m365_defender",
+        "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "host.user.name": "brent@elasticbv.onmicrosoft.com",
         "input.type": "log",
-        "log.offset": 16091,
+        "log.offset": 15990,
         "message": "Activity from infrequent country",
         "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com",
         "microsoft.m365_defender.alerts.classification": "FalsePositive",
         "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z",
         "microsoft.m365_defender.alerts.detectionSource": "MCAS",
-        "microsoft.m365_defender.alerts.entities.entityType": "Ip",
-        "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53",
+        "microsoft.m365_defender.alerts.entities.accountName": "brent",
+        "microsoft.m365_defender.alerts.entities.entityType": "User",
         "microsoft.m365_defender.alerts.incidentId": "14",
         "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType",
         "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z",
@@ -601,13 +599,18 @@
         "observer.name": "MicrosoftCloudAppSecurity",
         "observer.product": "365 Defender",
         "observer.vendor": "Microsoft",
+        "related.user": [
+            "brent@elasticbv.onmicrosoft.com"
+        ],
         "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.",
         "service.type": "microsoft",
         "tags": [
             "m365-defender",
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity"
+        "threat.technique.name": "SuspiciousActivity",
+        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "user.name": "brent@elasticbv.onmicrosoft.com"
     }
 ]