diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1aba526139d..5326fcc5930 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,6 +54,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adds oauth support for httpjson input. {issue}18415[18415] {pull}18892[18892] - Adds `split_events_by` option to httpjson input. {pull}19246[19246] - Adds `date_cursor` option to httpjson input. {pull}19483[19483] +- Adds Gsuite module with SAML support. {pull}19329[19329] +- Adds Gsuite User Accounts support. {pull}19329[19329] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 6e8112eb268..e63dff8db52 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -32,6 +32,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -23222,6 +23223,140 @@ type: keyword Latency as measured (for TCP flows only) during the time interval. This is the time elapsed between sending a SEQ and receiving a corresponding ACK and it contains the network RTT as well as the application related delay. +type: long + +-- + +[[exported-fields-gsuite]] +== gsuite fields + +gsuite Module + + + +[float] +=== gsuite + +Gsuite specific fields. +More information about specific fields can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + + + +*`gsuite.actor.type`*:: ++ +-- +The type of actor. +Values can be: + *USER*: Another user in the same domain. + *EXTERNAL_USER*: A user outside the domain. + *KEY*: A non-human actor. + + +type: keyword + +-- + +*`gsuite.actor.key`*:: ++ +-- +Only present when `actor.type` is `KEY`. Can be the `consumer_key` of the requestor for OAuth 2LO API requests or an identifier for robot accounts. + + +type: keyword + +-- + +*`gsuite.event.type`*:: ++ +-- +The type of GSuite event, mapped from `items[].events[].type` in the original payload. Each fileset can have a different set of values for it, more details can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + + +type: keyword + +example: audit#activity + +-- + +*`gsuite.kind`*:: ++ +-- +The type of API resource, mapped from `kind` in the original payload. More details can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + + +type: keyword + +example: audit#activity + +-- + +*`gsuite.organization.domain`*:: ++ +-- +The domain that is affected by the report's event. + + +type: keyword + +-- + + +*`gsuite.saml.application_name`*:: ++ +-- +Saml SP application name. + + +type: keyword + +-- + +*`gsuite.saml.failure_type`*:: ++ +-- +Login failure type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml. + + +type: keyword + +-- + +*`gsuite.saml.initiated_by`*:: ++ +-- +Requester of SAML authentication. + + +type: keyword + +-- + +*`gsuite.saml.orgunit_path`*:: ++ +-- +User orgunit. + + +type: keyword + +-- + +*`gsuite.saml.status_code`*:: ++ +-- +SAML status code. + + +type: long + +-- + +*`gsuite.saml.second_level_status_code`*:: ++ +-- +SAML second level status code. + + type: long -- diff --git a/filebeat/docs/modules/gsuite.asciidoc b/filebeat/docs/modules/gsuite.asciidoc new file mode 100644 index 00000000000..0870f008c79 --- /dev/null +++ b/filebeat/docs/modules/gsuite.asciidoc @@ -0,0 +1,107 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-gsuite]] +[role="xpack"] + +:modulename: gsuite +:has-dashboards: false + +== GSuite module + +beta[] + +This is a module for ingesting data from the different GSuite audit reports API's. + +include::../include/gs-link.asciidoc[] + +[float] +=== Compatibility + +It is compatible with a subset of applications under the https://developers.google.com/admin-sdk/reports/v1/get-start/getting-started[Google Reports API v1]. As of today it supports: + +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml[SAML Audit Activity Events] +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/user-accounts[User Accounts Activity Events] + +=== Configure the module + +In order for filebeat to ingest data from the Google Reports API you must set up a `ServiceAccount` that has access to the `Admin SDK API`. Additionally https://developers.google.com/admin-sdk/reports/v1/guides/delegation[Domain-Wide Delegation] is required for your application to work properly. + +This module will make use of the following `oauth2 scope`: + +- `https://www.googleapis.com/auth/admin.reports.audit.readonly` + +Once you have downloaded your service account credentials as a JSON file, +you can set up your module: + +[float] +===== Configuration options + +[source,yaml] +---- +- module: gsuite + saml: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" + user_accounts: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" +---- + +Every fileset has the following configuration options: + +*`var.jwt_file`*:: + +Specifies the path to the JWT credentials file. + +*`var.delegated_account`*:: + +Email of the admin user used to access the API. + +*`var.http_client_timeout`*:: + +Duration of the time limit on HTTP requests made by the module. Defaults to +`60s`. + +*`var.interval`*:: + +Duration between requests to the API. Defaults to `60s`. + +*`var.user_key`*:: + +Specifies the user key to fetch reports from. Defaults to `all`. + +[float] +==== GSuite Reports ECS fields + +This is a list of GSuite Reports fields that are mapped to ECS. + +[options="header"] +|======================================================================================= +| GSuite Reports | ECS Fields | +| items[].id.time | @timestamp | +| items[].id.uniqueQualifier | event.id | +| items[].id.applicationName | event.provider | +| items[].events[].name | event.action | +| items[].customerId | organization.id | +| items[].ipAddress | client.ip, related.ip, client.as.*, client.geo.* | +| items[].actor.email | client.user.email, client.user.name, client.user.domain | +| items[].actor.profileId | client.user.id | +|======================================================================================= + +These are the common ones to all filesets. + +:has-dashboards!: + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index dbf82900062..2fad0a66105 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -17,6 +17,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -63,6 +64,7 @@ include::modules/elasticsearch.asciidoc[] include::modules/envoyproxy.asciidoc[] include::modules/fortinet.asciidoc[] include::modules/googlecloud.asciidoc[] +include::modules/gsuite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/ibmmq.asciidoc[] include::modules/icinga.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 5bd64c486a0..7af1ee43ef7 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -547,6 +547,14 @@ filebeat.modules: # the subscription. var.credentials_file: ${path.config}/gcp-service-account-xyz.json +#-------------------------------- Gsuite Module -------------------------------- +- module: gsuite + # All logs + saml: + enabled: true + user_accounts: + enabled: true + #------------------------------- HAProxy Module ------------------------------- - module: haproxy # All logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index da50998f534..48ac4c4ce6c 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -28,6 +28,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/envoyproxy" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/fortinet" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/googlecloud" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/gsuite" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/ibmmq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/iptables" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/misp" diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index b9bdb45668c..6a09cf2fb92 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -50,9 +50,10 @@ type OAuth2 struct { TokenURL string `config:"token_url"` // google specific - GoogleCredentialsFile string `config:"google.credentials_file"` - GoogleCredentialsJSON []byte `config:"google.credentials_json"` - GoogleJWTFile string `config:"google.jwt_file"` + GoogleCredentialsFile string `config:"google.credentials_file"` + GoogleCredentialsJSON []byte `config:"google.credentials_json"` + GoogleJWTFile string `config:"google.jwt_file"` + GoogleDelegatedAccount string `config:"google.delegated_account"` // microsoft azure specific AzureTenantID string `config:"azure.tenant_id"` @@ -79,6 +80,15 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, } return creds.Client(ctx), nil case OAuth2ProviderGoogle: + if o.GoogleJWTFile != "" { + cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) + if err != nil { + return nil, fmt.Errorf("oauth2 client: error loading jwt credentials: %w", err) + } + cfg.Subject = o.GoogleDelegatedAccount + return cfg.Client(ctx), nil + } + creds, err := google.CredentialsFromJSON(ctx, o.GoogleCredentialsJSON, o.Scopes...) if err != nil { return nil, fmt.Errorf("oauth2 client: error loading credentials: %w", err) @@ -149,6 +159,9 @@ func (o *OAuth2) validateGoogleProvider() error { // credentials_json if len(o.GoogleCredentialsJSON) > 0 { + if o.GoogleDelegatedAccount != "" { + return errors.New("invalid configuration: google.delegated_account can only be provided with a jwt_file") + } if !json.Valid(o.GoogleCredentialsJSON) { return errors.New("invalid configuration: google.credentials_json must be valid JSON") } @@ -157,6 +170,9 @@ func (o *OAuth2) validateGoogleProvider() error { // credentials_file if o.GoogleCredentialsFile != "" { + if o.GoogleDelegatedAccount != "" { + return errors.New("invalid configuration: google.delegated_account can only be provided with a jwt_file") + } return o.populateCredentialsJSONFromFile(o.GoogleCredentialsFile) } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index a86c2aa76db..904702ee116 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -377,6 +377,29 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + { + name: "google must fail if the delegated_account is set without jwt_file", + expectedErr: "invalid configuration: google.delegated_account can only be provided with a jwt_file accessing 'oauth2'", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.credentials_file": "./testdata/credentials.json", + "google.delegated_account": "delegated@account.com", + }, + "url": "localhost", + }, + }, + { + name: "google must work with delegated_account and a valid jwt_file", + input: map[string]interface{}{ + "oauth2": map[string]interface{}{ + "provider": "google", + "google.jwt_file": "./testdata/credentials.json", + "google.delegated_account": "delegated@account.com", + }, + "url": "localhost", + }, + }, } for _, c := range cases { diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index fe65f63ebf8..3ed396db6c8 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -398,6 +398,9 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl } else { v, err = common.MapStr(obj).GetValue(in.config.JSONObjects) if err != nil { + if err == common.ErrKeyNotFound { + return nil + } return err } switch ts := v.(type) { diff --git a/x-pack/filebeat/module/gsuite/_meta/config.yml b/x-pack/filebeat/module/gsuite/_meta/config.yml new file mode 100644 index 00000000000..5301bb8567a --- /dev/null +++ b/x-pack/filebeat/module/gsuite/_meta/config.yml @@ -0,0 +1,6 @@ +- module: gsuite + # All logs + saml: + enabled: true + user_accounts: + enabled: true diff --git a/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc b/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc new file mode 100644 index 00000000000..4af4410b17e --- /dev/null +++ b/x-pack/filebeat/module/gsuite/_meta/docs.asciidoc @@ -0,0 +1,94 @@ +[role="xpack"] + +:modulename: gsuite +:has-dashboards: false + +== GSuite module + +beta[] + +This is a module for ingesting data from the different GSuite audit reports API's. + +include::../include/gs-link.asciidoc[] + +[float] +=== Compatibility + +It is compatible with a subset of applications under the https://developers.google.com/admin-sdk/reports/v1/get-start/getting-started[Google Reports API v1]. As of today it supports: + +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml[SAML Audit Activity Events] +- https://developers.google.com/admin-sdk/reports/v1/appendix/activity/user-accounts[User Accounts Activity Events] + +=== Configure the module + +In order for filebeat to ingest data from the Google Reports API you must set up a `ServiceAccount` that has access to the `Admin SDK API`. Additionally https://developers.google.com/admin-sdk/reports/v1/guides/delegation[Domain-Wide Delegation] is required for your application to work properly. + +This module will make use of the following `oauth2 scope`: + +- `https://www.googleapis.com/auth/admin.reports.audit.readonly` + +Once you have downloaded your service account credentials as a JSON file, +you can set up your module: + +[float] +===== Configuration options + +[source,yaml] +---- +- module: gsuite + saml: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" + user_accounts: + enabled: true + var.jwt_file: "./credentials_file.json" + var.delegated_account: "user@example.com" +---- + +Every fileset has the following configuration options: + +*`var.jwt_file`*:: + +Specifies the path to the JWT credentials file. + +*`var.delegated_account`*:: + +Email of the admin user used to access the API. + +*`var.http_client_timeout`*:: + +Duration of the time limit on HTTP requests made by the module. Defaults to +`60s`. + +*`var.interval`*:: + +Duration between requests to the API. Defaults to `60s`. + +*`var.user_key`*:: + +Specifies the user key to fetch reports from. Defaults to `all`. + +[float] +==== GSuite Reports ECS fields + +This is a list of GSuite Reports fields that are mapped to ECS. + +[options="header"] +|======================================================================================= +| GSuite Reports | ECS Fields | +| items[].id.time | @timestamp | +| items[].id.uniqueQualifier | event.id | +| items[].id.applicationName | event.provider | +| items[].events[].name | event.action | +| items[].customerId | organization.id | +| items[].ipAddress | client.ip, related.ip, client.as.*, client.geo.* | +| items[].actor.email | client.user.email, client.user.name, client.user.domain | +| items[].actor.profileId | client.user.id | +|======================================================================================= + +These are the common ones to all filesets. + +:has-dashboards!: + +:modulename!: diff --git a/x-pack/filebeat/module/gsuite/_meta/fields.yml b/x-pack/filebeat/module/gsuite/_meta/fields.yml new file mode 100644 index 00000000000..21ef9c6e692 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/_meta/fields.yml @@ -0,0 +1,42 @@ +- key: gsuite + title: "gsuite" + description: > + gsuite Module + fields: + - name: gsuite + default_field: false + type: group + description: > + Gsuite specific fields. + + More information about specific fields can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + fields: + - name: actor.type + type: keyword + description: > + The type of actor. + + Values can be: + *USER*: Another user in the same domain. + *EXTERNAL_USER*: A user outside the domain. + *KEY*: A non-human actor. + - name: actor.key + type: keyword + description: > + Only present when `actor.type` is `KEY`. Can be the `consumer_key` of the requestor for OAuth 2LO API requests or an identifier for robot accounts. + - name: event.type + type: keyword + description: > + The type of GSuite event, mapped from `items[].events[].type` in the original payload. + Each fileset can have a different set of values for it, more details can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + example: audit#activity + - name: kind + type: keyword + description: > + The type of API resource, mapped from `kind` in the original payload. + More details can be found at https://developers.google.com/admin-sdk/reports/v1/reference/activities/list + example: audit#activity + - name: organization.domain + type: keyword + description: > + The domain that is affected by the report's event. diff --git a/x-pack/filebeat/module/gsuite/config/common.js b/x-pack/filebeat/module/gsuite/config/common.js new file mode 100644 index 00000000000..fd6955f98a9 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/config/common.js @@ -0,0 +1,83 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +var gsuite = (function () { + var processor = require("processor"); + + var decodeJson = new processor.DecodeJSONFields({ + fields: ["message"], + target: "json", + }); + + var parseTimestamp = new processor.Timestamp({ + field: "json.id.time", + timezone: "UTC", + layouts: ["2006-01-02T15:04:05.999Z"], + tests: ["2020-02-05T18:19:23.599Z"], + ignore_missing: true, + }); + + var convertFields = new processor.Convert({ + fields: [ + { from: "message", to: "event.original" }, + { from: "json.events.name", to: "event.action" }, + { from: "json.id.applicationName", to: "event.provider" }, + { from: "json.id.uniqueQualifier", to: "event.id", type: "string" }, + { from: "json.actor.email", to: "client.user.email" }, + { from: "json.actor.profileId", to: "client.user.id", type: "string" }, + { from: "json.ipAddress", to: "client.ip", type: "ip" }, + { from: "json.kind", to: "gsuite.kind" }, + { from: "json.id.customerId", to: "organization.id", type: "string" }, + { from: "json.actor.callerType", to: "gsuite.actor.type" }, + { from: "json.actor.key", to: "gsuite.actor.key" }, + { from: "json.ownerDomain", to: "gsuite.organization.domain" }, + { from: "json.events.type", to: "gsuite.event.type" }, + ], + mode: "rename", + ignore_missing: true, + fail_on_error: false, + }); + + var completeUserData = function(evt) { + var email = evt.Get("client.user.email"); + if (!email) { + return; + } + + var data = email.split("@"); + if (data.length !== 2) { + return; + } + + evt.Put("client.user.name", data[0]); + evt.Put("client.user.domain", data[1]); + }; + + var copyFields = function(evt) { + var ip = evt.Get("client.ip"); + if (ip) { + evt.Put("related.ip", [ip]); + } + var userName = evt.Get("client.user.name"); + if (userName) { + evt.Put("related.user", [userName]); + } + }; + + var pipeline = new processor.Chain() + .Add(decodeJson) + .Add(parseTimestamp) + .Add(convertFields) + .Add(completeUserData) + .Add(copyFields) + .Build(); + + return { + process: pipeline.Run, + }; +}()); + +function process(evt) { + return gsuite.process(evt); +} diff --git a/x-pack/filebeat/module/gsuite/fields.go b/x-pack/filebeat/module/gsuite/fields.go new file mode 100644 index 00000000000..5003648f341 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package gsuite + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "gsuite", asset.ModuleFieldsPri, AssetGsuite); err != nil { + panic(err) + } +} + +// AssetGsuite returns asset data. +// This is the base64 encoded gzipped contents of module/gsuite. +func AssetGsuite() string { + return "eJzMVU2PIzUQvedXlJYD0orpCI45IEUorNDOMKvJLmKFUKfSLqdLcbsau5yd5tcju5Mhk2TExwxofWq5vl49v6q+gi0NM9jExEoTAGV1NINX48WrCYCh2ATulcXP4NsJAOy94UZMcjnIMjkTZ8V2BR47OsqYjyGLyWldHGdg0cWDSYc+ewdJ/YPzWcF83oxFY08NW272RavJg8ONBAL2VkKHORhwLUlPA6BBD2sCK8kbQIVWtY+z6dTQjpz0FGK1Edk4qhrppmg69lfRbKeBegkap7uvp4EsBfINTbFR3rEyxanjqHssx3wcc4KNSqhyxw+mAwFbGj5JMEf3T9CQz/uWShiI3eecPLL/hC7RodPZIxPA6w/Lxd3rGcy9aEsBUqQA7EFbgogdgZEO2VenYYuf3y/ufpxf14f4MVKSRjZUwp+IfLv4WPy9+Ks2degPoC/zs6XhefTcejdAHyiSV/jUkofVn8yvgCOs3i4+rir4bpRChr5qxMfUUai3NKwysfk20G+JokoAKwFu50lb+Ob6FubvfjjYIkgA9MCGvLJlGn2DrEUBm0aS13jeKu3I6wtL4c2yTEhJ/RV02PdkwAbpYMVKXfzl16rY8seeivHZJfCGPTrocXCC5vELLrBpwbKjSFo01eKOAMGwLWOgkA1iYTfKLrfPGUCeR0OK7P6XqcuH7rHr8wLDZFi/2DsOZ/Rv2ZuXI36UQ5QUGjohPhf6mzzffMZ8Sdig59/LXq3GMX8efSOBYybQFjWPJVpLjZKB9bAfv9zMl3E/LWeoInbuDMbxrwQuLOPjBNj3jpvSVZ1vTiA+1ddf9gawxM7B8t1xhVKzuojDIrsUqD7ZB8/EcC0b9ofcJVEF3+dtBVkFWbi9xMhrR4fRLZIBlX8jt6x6b/j+oLZhmp/ncr/sWRmVTL0eXq7fu3Ej55+SheX85howaZvX8vgAl7FI2CTPWveo7cthAfhQ/o5j8suVo6KmWDdiLj+6E7/5h6rLTY9pIad9oi414k3t8tvW/xWIUgNKjceI/ggAAP//yAgQFA==" +} diff --git a/x-pack/filebeat/module/gsuite/ingest/common.yml b/x-pack/filebeat/module/gsuite/ingest/common.yml new file mode 100644 index 00000000000..46ae0a7aa25 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/ingest/common.yml @@ -0,0 +1,30 @@ +description: Pipeline for parsing gsuite logs +processors: + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + - remove: + field: json + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/gsuite/saml/_meta/fields.yml b/x-pack/filebeat/module/gsuite/saml/_meta/fields.yml new file mode 100644 index 00000000000..b7e9efc0926 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/_meta/fields.yml @@ -0,0 +1,27 @@ +- name: saml + type: group + fields: + - name: application_name + type: keyword + description: > + Saml SP application name. + - name: failure_type + type: keyword + description: > + Login failure type. For a list of possible values refer to https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml. + - name: initiated_by + type: keyword + description: > + Requester of SAML authentication. + - name: orgunit_path + type: keyword + description: > + User orgunit. + - name: status_code + type: long + description: > + SAML status code. + - name: second_level_status_code + type: long + description: > + SAML second level status code. diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml new file mode 100644 index 00000000000..4139260481a --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/config/config.yml @@ -0,0 +1,48 @@ +{{ if eq .input "httpjson" }} +type: httpjson + +url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user_key }}/applications/saml +json_objects_array: items +split_events_by: events + +interval: {{ .interval }} + +{{ if .http_client_timeout }} +http_client_timeout: {{ .http_client_timeout }} +{{ end }} + +oauth2.provider: google +oauth2.google.jwt_file: {{ .jwt_file }} +oauth2.google.delegated_account: {{ .delegated_account }} +oauth2.scopes: + - https://www.googleapis.com/auth/admin.reports.audit.readonly + +date_cursor.field: id.time +date_cursor.url_field: startTime +date_cursor.initial_interval: {{ .initial_interval }} + +{{ else if eq .input "file" }} +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 + - script: + lang: javascript + id: gsuite-common + file: ${path.home}/module/gsuite/config/common.js + - script: + lang: javascript + id: gsuite-saml + file: ${path.home}/module/gsuite/saml/config/pipeline.js diff --git a/x-pack/filebeat/module/gsuite/saml/config/pipeline.js b/x-pack/filebeat/module/gsuite/saml/config/pipeline.js new file mode 100644 index 00000000000..3ad58062823 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/config/pipeline.js @@ -0,0 +1,60 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +var saml = (function () { + var processor = require("processor"); + + var categorizeEvent = function(evt) { + evt.Put("event.type", ["start"]); + evt.Put("event.category", ["authentication"]); + switch (evt.Get("event.action")) { + case "login_failure": + evt.Put("event.outcome", "failure"); + break; + case "login_success": + evt.Put("event.outcome", "success"); + break; + } + }; + + var processParams = function(evt) { + var params = evt.Get("json.events.parameters"); + if (!params || !Array.isArray(params)) { + return; + } + + var prefixRegex = /^(saml_)/; + + params.forEach(function(p){ + p.name = p.name.replace(prefixRegex, ""); + + // all saml event parameters are strings. + // for this reason we know for sure they are in the 'value' field. + // https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml + switch (p.name) { + case "status_code": + case "second_level_status_code": + evt.Put("gsuite.saml."+p.name, parseInt(p.value)); + break; + default: + evt.Put("gsuite.saml."+p.name, p.value); + } + }); + + evt.Delete("json.events.parameters"); + }; + + var pipeline = new processor.Chain() + .Add(categorizeEvent) + .Add(processParams) + .Build(); + + return { + process: pipeline.Run, + }; +}()); + +function process(evt) { + return saml.process(evt); +} diff --git a/x-pack/filebeat/module/gsuite/saml/manifest.yml b/x-pack/filebeat/module/gsuite/saml/manifest.yml new file mode 100644 index 00000000000..7b6e1e42ced --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/manifest.yml @@ -0,0 +1,24 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: jwt_file + - name: delegated_account + - name: initial_interval + default: 24h + - name: http_client_timeout + default: 60s + - name: user_key + default: all + - name: interval + default: 5s + - name: tags + default: [forwarded] + +input: config/config.yml +ingest_pipeline: ../ingest/common.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log b/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log new file mode 100644 index 00000000000..678193e25d5 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log @@ -0,0 +1,2 @@ +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"saml","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_failure","parameters":[{"name":"application_name","value":"app"},{"name":"failure_type","value":"failure_app_not_configured_for_user"},{"name":"initiated_by","value":"idp"},{"name":"orgunit_path","value":"ounit"},{"name":"saml_second_level_status_code","value":"400"},{"name":"saml_status_code","value":"400"}]}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:01Z","uniqueQualifier":1,"applicationName":"saml","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"login","name":"login_success","parameters":[{"name":"application_name","value":"app"},{"name":"initiated_by","value":"idp"},{"name":"orgunit_path","value":"ounit"},{"name":"saml_status_code","value":"400"}]}} diff --git a/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log-expected.json new file mode 100644 index 00000000000..aa57092897b --- /dev/null +++ b/x-pack/filebeat/module/gsuite/saml/test/gsuite-saml-test.json.log-expected.json @@ -0,0 +1,108 @@ +[ + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_failure", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.saml", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"saml\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_failure\",\"parameters\":[{\"name\":\"application_name\",\"value\":\"app\"},{\"name\":\"failure_type\",\"value\":\"failure_app_not_configured_for_user\"},{\"name\":\"initiated_by\",\"value\":\"idp\"},{\"name\":\"orgunit_path\",\"value\":\"ounit\"},{\"name\":\"saml_second_level_status_code\",\"value\":\"400\"},{\"name\":\"saml_status_code\",\"value\":\"400\"}]}}", + "event.outcome": "failure", + "event.provider": "saml", + "event.type": [ + "start" + ], + "fileset.name": "saml", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "gsuite.saml.application_name": "app", + "gsuite.saml.failure_type": "failure_app_not_configured_for_user", + "gsuite.saml.initiated_by": "idp", + "gsuite.saml.orgunit_path": "ounit", + "gsuite.saml.second_level_status_code": 400, + "gsuite.saml.status_code": 400, + "input.type": "log", + "log.offset": 0, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:01.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "login_success", + "event.category": [ + "authentication" + ], + "event.dataset": "gsuite.saml", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:01Z\",\"uniqueQualifier\":1,\"applicationName\":\"saml\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"login\",\"name\":\"login_success\",\"parameters\":[{\"name\":\"application_name\",\"value\":\"app\"},{\"name\":\"initiated_by\",\"value\":\"idp\"},{\"name\":\"orgunit_path\",\"value\":\"ounit\"},{\"name\":\"saml_status_code\",\"value\":\"400\"}]}}", + "event.outcome": "success", + "event.provider": "saml", + "event.type": [ + "start" + ], + "fileset.name": "saml", + "gsuite.actor.type": "USER", + "gsuite.event.type": "login", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "gsuite.saml.application_name": "app", + "gsuite.saml.initiated_by": "idp", + "gsuite.saml.orgunit_path": "ounit", + "gsuite.saml.status_code": 400, + "input.type": "log", + "log.offset": 606, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml new file mode 100644 index 00000000000..88afbc0d629 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml @@ -0,0 +1,48 @@ +{{ if eq .input "httpjson" }} +type: httpjson + +url: https://www.googleapis.com/admin/reports/v1/activity/users/{{ .user_key }}/applications/user_accounts +json_objects_array: items +split_events_by: events + +interval: {{ .interval }} + +{{ if .http_client_timeout }} +http_client_timeout: {{ .http_client_timeout }} +{{ end }} + +oauth2.provider: google +oauth2.google.jwt_file: {{ .jwt_file }} +oauth2.google.delegated_account: {{ .delegated_account }} +oauth2.scopes: + - https://www.googleapis.com/auth/admin.reports.audit.readonly + +date_cursor.field: id.time +date_cursor.url_field: startTime +date_cursor.initial_interval: {{ .initial_interval }} + +{{ else if eq .input "file" }} +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 + - script: + lang: javascript + id: gsuite-common + file: ${path.home}/module/gsuite/config/common.js + - script: + lang: javascript + id: gsuite-user_accounts + file: ${path.home}/module/gsuite/user_accounts/config/pipeline.js diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/pipeline.js b/x-pack/filebeat/module/gsuite/user_accounts/config/pipeline.js new file mode 100644 index 00000000000..89b54fa72db --- /dev/null +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/pipeline.js @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +var userAccounts = (function () { + var processor = require("processor"); + + var categorizeEvent = function(evt) { + evt.Put("event.type", ["change", "user"]); + evt.Put("event.category", ["iam"]); + }; + + var pipeline = new processor.Chain() + .Add(categorizeEvent) + .Build(); + + return { + process: pipeline.Run, + }; +}()); + +function process(evt) { + return userAccounts.process(evt); +} diff --git a/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml b/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml new file mode 100644 index 00000000000..7b6e1e42ced --- /dev/null +++ b/x-pack/filebeat/module/gsuite/user_accounts/manifest.yml @@ -0,0 +1,24 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: jwt_file + - name: delegated_account + - name: initial_interval + default: 24h + - name: http_client_timeout + default: 60s + - name: user_key + default: all + - name: interval + default: 5s + - name: tags + default: [forwarded] + +input: config/config.yml +ingest_pipeline: ../ingest/common.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log b/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log new file mode 100644 index 00000000000..7da8fdec935 --- /dev/null +++ b/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log @@ -0,0 +1,8 @@ +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"2sv_change","name":"2sv_disable"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"2sv_change","name":"2sv_enroll"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"password_change","name":"password_edit"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"recovery_info_change","name":"recovery_email_edit"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"recovery_info_change","name":"recovery_phone_edit"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"recovery_info_change","name":"recovery_secret_qa_edit"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"titanium_change","name":"titanium_enroll"}} +{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"user_accounts","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"98.235.162.24","events":{"type":"titanium_change","name":"titanium_unenroll"}} diff --git a/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log-expected.json b/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log-expected.json new file mode 100644 index 00000000000..a4b00539c9f --- /dev/null +++ b/x-pack/filebeat/module/gsuite/user_accounts/test/gsuite-user_accounts-test.json.log-expected.json @@ -0,0 +1,386 @@ +[ + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "2sv_disable", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"2sv_change\",\"name\":\"2sv_disable\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "2sv_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 0, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "2sv_enroll", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"2sv_change\",\"name\":\"2sv_enroll\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "2sv_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 316, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "password_edit", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"password_change\",\"name\":\"password_edit\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "password_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 631, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "recovery_email_edit", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_email_edit\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "recovery_info_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 954, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "recovery_phone_edit", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_phone_edit\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "recovery_info_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1288, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "recovery_secret_qa_edit", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"recovery_info_change\",\"name\":\"recovery_secret_qa_edit\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "recovery_info_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1622, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "titanium_enroll", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"titanium_change\",\"name\":\"titanium_enroll\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "titanium_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 1960, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-10-02T15:00:00.000Z", + "client.as.number": 7922, + "client.as.organization.name": "Comcast Cable Communications, LLC", + "client.geo.city_name": "State College", + "client.geo.continent_name": "North America", + "client.geo.country_iso_code": "US", + "client.geo.location.lat": 40.7957, + "client.geo.location.lon": -77.8618, + "client.geo.region_iso_code": "US-PA", + "client.geo.region_name": "Pennsylvania", + "client.ip": "98.235.162.24", + "client.user.domain": "bar.com", + "client.user.email": "foo@bar.com", + "client.user.id": "1", + "client.user.name": "foo", + "event.action": "titanium_unenroll", + "event.category": [ + "iam" + ], + "event.dataset": "gsuite.user_accounts", + "event.id": "1", + "event.module": "gsuite", + "event.original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"user_accounts\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"98.235.162.24\",\"events\":{\"type\":\"titanium_change\",\"name\":\"titanium_unenroll\"}}", + "event.provider": "user_accounts", + "event.type": [ + "change", + "user" + ], + "fileset.name": "user_accounts", + "gsuite.actor.type": "USER", + "gsuite.event.type": "titanium_change", + "gsuite.kind": "admin#reports#activity", + "gsuite.organization.domain": "elastic.com", + "input.type": "log", + "log.offset": 2285, + "organization.id": "1", + "related.ip": [ + "98.235.162.24" + ], + "related.user": [ + "foo" + ], + "service.type": "gsuite", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/gsuite.yml.disabled b/x-pack/filebeat/modules.d/gsuite.yml.disabled new file mode 100644 index 00000000000..efb79107ef4 --- /dev/null +++ b/x-pack/filebeat/modules.d/gsuite.yml.disabled @@ -0,0 +1,9 @@ +# Module: gsuite +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-gsuite.html + +- module: gsuite + # All logs + saml: + enabled: true + user_accounts: + enabled: true