From 35f962526597ca7b4592b5d30b3566d770f06f41 Mon Sep 17 00:00:00 2001 From: Apoorva Joshi <30438249+ajosh0504@users.noreply.github.com> Date: Mon, 14 Jun 2021 12:36:08 -0700 Subject: [PATCH] [ML] Adds Authentication module with six ML jobs for ECS data (Auditbeat, Winlogbeat, Filebeat and Logs) (#101840) * Adding Security Authentication jobs in 7.14 * Renamed some jobs * Changing memory limits and linting change * Linting fix * Changed the order * Adding module to ml_modules.tsx * Update recognize_module.ts this test modules uses older Auditbeat data which predates the event.category field so the test has to be skipped per https://elastic.zoom.us/j/93000943632?pwd=TmpvNWhtYUNzMUc0c0N6Tlc2QlVPZz09 * Update recognize_module.ts needs to be a single line * Update recognize_module.ts Some linters want spaces and some linters want no spaces. This linter wants spaces. * descriptions added description text * Update auth_rare_hour_for_a_user.json removed a wayward newline char * Minor nitpicking * memory limits raised memory limits to 128mb which is larger than the highest observed peak model bytes for the most memory hungry jobs in this event class. Co-authored-by: Craig Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../modules/security_auth/logo.json | 3 + .../modules/security_auth/manifest.json | 77 +++++++++++++++++++ .../ml/auth_high_count_logon_events.json | 29 +++++++ ...gh_count_logon_events_for_a_source_ip.json | 34 ++++++++ .../ml/auth_high_count_logon_fails.json | 29 +++++++ .../ml/auth_rare_hour_for_a_user.json | 33 ++++++++ .../ml/auth_rare_source_ip_for_a_user.json | 34 ++++++++ .../security_auth/ml/auth_rare_user.json | 33 ++++++++ ...datafeed_auth_high_count_logon_events.json | 26 +++++++ ...gh_count_logon_events_for_a_source_ip.json | 26 +++++++ .../datafeed_auth_high_count_logon_fails.json | 26 +++++++ .../datafeed_auth_rare_hour_for_a_user.json | 26 +++++++ ...tafeed_auth_rare_source_ip_for_a_user.json | 26 +++++++ .../ml/datafeed_auth_rare_user.json | 26 +++++++ .../components/ml_popover/ml_modules.tsx | 1 + .../apis/ml/modules/get_module.ts | 1 + .../apis/ml/modules/recognize_module.ts | 5 +- 17 files changed, 433 insertions(+), 2 deletions(-) create mode 100755 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json create mode 100755 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_source_ip_for_a_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_user.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json new file mode 100755 index 00000000000000..862f970b7405db --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "logoSecurity" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json new file mode 100755 index 00000000000000..480f49f3f2b198 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json @@ -0,0 +1,77 @@ +{ + "id": "security_auth", + "title": "Security: Authentication", + "description": "Detect anomalous activity in your ECS-compatible authentication logs.", + "type": "auth data", + "logoFile": "logo.json", + "defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + } + ] + } + }, + "jobs": [ + { + "id": "auth_high_count_logon_events_for_a_source_ip", + "file": "auth_high_count_logon_events_for_a_source_ip.json" + }, + { + "id": "auth_high_count_logon_fails", + "file": "auth_high_count_logon_fails.json" + }, + { + "id": "auth_high_count_logon_events", + "file": "auth_high_count_logon_events.json" + }, + { + "id": "auth_rare_hour_for_a_user", + "file": "auth_rare_hour_for_a_user.json" + }, + { + "id": "auth_rare_source_ip_for_a_user", + "file": "auth_rare_source_ip_for_a_user.json" + }, + { + "id": "auth_rare_user", + "file": "auth_rare_user.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-auth_high_count_logon_events_for_a_source_ip", + "file": "datafeed_auth_high_count_logon_events_for_a_source_ip.json", + "job_id": "auth_high_count_logon_events_for_a_source_ip" + }, + { + "id": "datafeed-auth_high_count_logon_fails", + "file": "datafeed_auth_high_count_logon_fails.json", + "job_id": "auth_high_count_logon_fails" + }, + { + "id": "datafeed-auth_high_count_logon_events", + "file": "datafeed_auth_high_count_logon_events.json", + "job_id": "auth_high_count_logon_events" + }, + { + "id": "datafeed-auth_rare_hour_for_a_user", + "file": "datafeed_auth_rare_hour_for_a_user.json", + "job_id": "auth_rare_hour_for_a_user" + }, + { + "id": "datafeed-auth_rare_source_ip_for_a_user", + "file": "datafeed_auth_rare_source_ip_for_a_user.json", + "job_id": "auth_rare_source_ip_for_a_user" + }, + { + "id": "datafeed-auth_rare_user", + "file": "datafeed_auth_rare_user.json", + "job_id": "auth_rare_user" + } + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json new file mode 100644 index 00000000000000..ee84fb222bb5c7 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json @@ -0,0 +1,29 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for an unusually large spike in successful authentication events. This can be due to password spraying, user enumeration or brute force activity.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high count of logon events", + "function": "high_non_zero_count", + "detector_index": 0 + } + ], + "influencers": [] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json new file mode 100644 index 00000000000000..7bbbc81b6de7ab --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for an unusually large spike in successful authentication events events from a particular source IP address. This can be due to password spraying, user enumeration or brute force activity.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high count of auth events for a source IP", + "function": "high_non_zero_count", + "by_field_name": "source.ip", + "detector_index": 0 + } + ], + "influencers": [ + "source.ip", + "winlog.event_data.LogonType", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json new file mode 100644 index 00000000000000..4b7094e92c6ecf --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json @@ -0,0 +1,29 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for an unusually large spike in authentication failure events. This can be due to password spraying, user enumeration or brute force activity and may be a precursor to account takeover or credentialed access.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high count of logon fails", + "function": "high_non_zero_count", + "detector_index": 0 + } + ], + "influencers": [] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json new file mode 100644 index 00000000000000..bb86d256e59df0 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json @@ -0,0 +1,33 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for a user logging in at a time of day that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different time zones. In addition, unauthorized user activity often takes place during non-business hours.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare hour for a user", + "function": "time_of_day", + "by_field_name": "user.name", + "detector_index": 0 + } + ], + "influencers": [ + "source.ip", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json new file mode 100644 index 00000000000000..6f72e148fa38ed --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json @@ -0,0 +1,34 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for a user logging in from an IP address that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different locations. An unusual source IP address for a username could also be due to lateral movement when a compromised account is used to pivot between hosts.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare source IP for a user", + "function": "rare", + "by_field_name": "source.ip", + "partition_field_name": "user.name", + "detector_index": 0 + } + ], + "influencers": [ + "source.ip", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json new file mode 100644 index 00000000000000..5cb9c7112b29d3 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json @@ -0,0 +1,33 @@ +{ + "job_type": "anomaly_detector", + "description": "Security: Authentication - looks for an unusual user name in the authentication logs. An unusual user name is one way of detecting credentialed access by means of a new or dormant user account. A user account that is normally inactive, because the user has left the organization, which becomes active, may be due to credentialed access using a compromised account password. Threat actors will sometimes also create new users as a means of persisting in a compromised web application.", + "groups": [ + "security", + "authentication" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare user", + "function": "rare", + "by_field_name": "user.name", + "detector_index": 0 + } + ], + "influencers": [ + "source.ip", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-auth" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json new file mode 100644 index 00000000000000..eb81179e443637 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_high_count_logon_events", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json new file mode 100644 index 00000000000000..dfed3ada1fe0be --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_high_count_logon_events_for_a_source_ip", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json new file mode 100644 index 00000000000000..431c115b34d604 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_high_count_logon_fails", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "failure" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json new file mode 100644 index 00000000000000..377197231f28c1 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_rare_hour_for_a_user", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_source_ip_for_a_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_source_ip_for_a_user.json new file mode 100644 index 00000000000000..dfa2ad7ab397c9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_source_ip_for_a_user.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_rare_source_ip_for_a_user", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_user.json new file mode 100644 index 00000000000000..f7de5d3aee71a7 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_user.json @@ -0,0 +1,26 @@ +{ + "job_id": "auth_rare_user", + "indices": [ + "auditbeat-*", + "logs-*", + "filebeat-*", + "winlogbeat-*" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "authentication" + } + }, + { + "term": { + "event.outcome": "success" + } + } + ] + } + } +} \ No newline at end of file diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx index 8dac6234f19a86..e7199f6df2b1f5 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx @@ -17,6 +17,7 @@ export const mlModules: string[] = [ 'siem_packetbeat', 'siem_winlogbeat', 'siem_winlogbeat_auth', + 'security_auth', 'security_linux', 'security_network', 'security_windows', diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index 4fa79b915cc5df..0a3e2dbed570bf 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -30,6 +30,7 @@ const moduleIds = [ 'nginx_ecs', 'sample_data_ecommerce', 'sample_data_weblogs', + 'security_auth', 'security_linux', 'security_network', 'security_windows', diff --git a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts index 2181bea8b40407..2742fbff294c0d 100644 --- a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts @@ -84,7 +84,7 @@ export default ({ getService }: FtrProviderContext) => { user: USER.ML_POWERUSER, expected: { responseCode: 200, - moduleIds: ['siem_auditbeat', 'siem_auditbeat_auth'], + moduleIds: ['security_auth', 'siem_auditbeat', 'siem_auditbeat_auth'], }, }, { @@ -105,6 +105,7 @@ export default ({ getService }: FtrProviderContext) => { expected: { responseCode: 200, moduleIds: [ + 'security_auth', 'security_network', 'security_windows', 'siem_winlogbeat', @@ -148,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => { user: USER.ML_POWERUSER, expected: { responseCode: 200, - moduleIds: ['security_linux', 'security_network', 'security_windows'], + moduleIds: ['security_auth', 'security_linux', 'security_network', 'security_windows'], }, }, {