Add support for platform logs in Filebeat Azure module (#22371) (#22749)

* mofidy doc

* platformlogs

* fix

* separate pr

* work on platform

* work

* platforms

* changelog

* fix file

* add tests

* add mapping

* test

* update mapping

* fix file name

* update file

* map field

* update files

* fix logs

* generate tests

(cherry picked from commit 2ae52c3)

CHANGELOG.next.asciidoc

@@ -516,6 +516,7 @@ from being added to events by default. {pull}18159[18159]
 - Copy tag names from MISP data into events. {pull}21664[21664]
 - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
 - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]
+- Add platform logs in the azure filebeat module. {pull}22371[22371]
 - Added `event.ingested` field to data from the Netflow module. {pull}22412[22412]
 - Improve panw ECS url fields mapping. {pull}22481[22481]
 - Improve Nats filebeat dashboard. {pull}22726[22726]

filebeat/docs/fields.asciidoc

@@ -3055,6 +3055,153 @@ type: keyword
 
 --
 
+[float]
+=== platformlogs
+
+Fields for Azure platform logs.
+
+
+
+*`azure.platformlogs.operation_name`*::
++
+--
+Operation name
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.result_type`*::
++
+--
+Result type
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.result_signature`*::
++
+--
+Result signature
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.category`*::
++
+--
+Category
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.event_category`*::
++
+--
+Event Category
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.status`*::
++
+--
+Status
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.ccpNamespace`*::
++
+--
+ccpNamespace
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.Cloud`*::
++
+--
+Cloud
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.Environment`*::
++
+--
+Environment
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.EventTimeString`*::
++
+--
+EventTimeString
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.Caller`*::
++
+--
+Caller
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.ScaleUnit`*::
++
+--
+ScaleUnit
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.ActivityId`*::
++
+--
+ActivityId
+
+
+type: keyword
+
+--
+
+*`azure.platformlogs.properties.*`*::
++
+--
+Properties
+
+
+type: object
+
+--
+
 [float]
 === signinlogs
 

filebeat/docs/modules/azure.asciidoc

@@ -24,6 +24,9 @@ The module contains the following filesets:
 `activitylogs` ::
 Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription.
 
+`platformlogs` ::
+Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on.
+
 `signinlogs` ::
 Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities.
 
@@ -46,6 +49,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
       storage_account_key: ""
       resource_manager_endpoint: ""
 
+   platformlogs:
+     enabled: false
+     var:
+       eventhub: ""
+       consumer_group: "$Default"
+       connection_string: ""
+       storage_account: ""
+       storage_account_key: ""
+       resource_manager_endpoint: ""
+
    auditlogs:
      enabled: false
      var:

x-pack/filebeat/filebeat.reference.yml

@@ -365,6 +365,16 @@ filebeat.modules:
       # the storage account key, this key will be used to authorize access to data in your storage account
       storage_account_key: ""
 
+  platformlogs:
+    enabled: false
+  #  var:
+  #    eventhub: ""
+  #    consumer_group: "$Default"
+  #    connection_string: ""
+  #    storage_account: ""
+  #    storage_account_key: ""
+
+
   auditlogs:
     enabled: false
  #   var:

x-pack/filebeat/module/azure/_meta/config.yml

@@ -14,6 +14,16 @@
       # the storage account key, this key will be used to authorize access to data in your storage account
       storage_account_key: ""
 
+  platformlogs:
+    enabled: false
+  #  var:
+  #    eventhub: ""
+  #    consumer_group: "$Default"
+  #    connection_string: ""
+  #    storage_account: ""
+  #    storage_account_key: ""
+
+
   auditlogs:
     enabled: false
  #   var:

x-pack/filebeat/module/azure/_meta/docs.asciidoc

@@ -19,6 +19,9 @@ The module contains the following filesets:
 `activitylogs` ::
 Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription.
 
+`platformlogs` ::
+Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on.
+
 `signinlogs` ::
 Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities.
 
@@ -41,6 +44,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi
       storage_account_key: ""
       resource_manager_endpoint: ""
 
+   platformlogs:
+     enabled: false
+     var:
+       eventhub: ""
+       consumer_group: "$Default"
+       connection_string: ""
+       storage_account: ""
+       storage_account_key: ""
+       resource_manager_endpoint: ""
+
    auditlogs:
      enabled: false
      var:

x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml

@@ -0,0 +1,66 @@
+- name: platformlogs
+  type: group
+  release: beta
+  default_field: false
+  description: >
+    Fields for Azure platform logs.
+  fields:
+    - name: operation_name
+      type: keyword
+      description: >
+        Operation name
+    - name: result_type
+      type: keyword
+      description: >
+        Result type
+    - name: result_signature
+      type: keyword
+      description: >
+        Result signature
+    - name: category
+      type: keyword
+      description: >
+        Category
+    - name: event_category
+      type: keyword
+      description: >
+        Event Category
+    - name: status
+      type: keyword
+      description: >
+        Status
+    - name: ccpNamespace
+      type: keyword
+      description: >
+        ccpNamespace
+    - name: Cloud
+      type: keyword
+      description: >
+        Cloud
+    - name: Environment
+      type: keyword
+      description: >
+        Environment
+    - name: EventTimeString
+      type: keyword
+      description: >
+        EventTimeString
+    - name: Caller
+      type: keyword
+      description: >
+        Caller
+    - name: ScaleUnit
+      type: keyword
+      description: >
+        ScaleUnit
+    - name: ActivityId
+      type: keyword
+      description: >
+        ActivityId
+    - name: properties.*
+      type: object
+      object_type: keyword
+      object_type_mapping_type: "*"
+      description: >
+        Properties
+

x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml

@@ -0,0 +1,16 @@
+type: azure-eventhub
+connection_string: {{ .connection_string }}
+eventhub: {{ .eventhub }}
+consumer_group: {{ .consumer_group }}
+storage_account: {{ .storage_account }}
+storage_account_key: {{ .storage_account_key }}
+resource_manager_endpoint: {{ .resource_manager_endpoint }}
+storage_account_container: filebeat-platformlogs-{{ .eventhub }}
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.6.0

x-pack/filebeat/module/azure/platformlogs/config/file.yml

@@ -0,0 +1,14 @@
+type: log
+paths:
+  {{ range $i, $path := .paths }}
+- {{$path}}
+  {{ end }}
+exclude_files: [".gz$"]
+tags: {{.tags | tojson}}
+publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.6.0