-
Notifications
You must be signed in to change notification settings - Fork 8.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] Adds Authentication module with six ML jobs for ECS data (Auditb…
…eat, Winlogbeat, Filebeat and Logs) (#101840) * Adding Security Authentication jobs in 7.14 * Renamed some jobs * Changing memory limits and linting change * Linting fix * Changed the order * Adding module to ml_modules.tsx * Update recognize_module.ts this test modules uses older Auditbeat data which predates the event.category field so the test has to be skipped per https://elastic.zoom.us/j/93000943632?pwd=TmpvNWhtYUNzMUc0c0N6Tlc2QlVPZz09 * Update recognize_module.ts needs to be a single line * Update recognize_module.ts Some linters want spaces and some linters want no spaces. This linter wants spaces. * descriptions added description text * Update auth_rare_hour_for_a_user.json removed a wayward newline char * Minor nitpicking * memory limits raised memory limits to 128mb which is larger than the highest observed peak model bytes for the most memory hungry jobs in this event class. Co-authored-by: Craig <mailredirector36@gmail.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
- Loading branch information
1 parent
8bdc0d7
commit 35f9625
Showing
17 changed files
with
433 additions
and
2 deletions.
There are no files selected for viewing
3 changes: 3 additions & 0 deletions
3
x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"icon": "logoSecurity" | ||
} |
77 changes: 77 additions & 0 deletions
77
x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
{ | ||
"id": "security_auth", | ||
"title": "Security: Authentication", | ||
"description": "Detect anomalous activity in your ECS-compatible authentication logs.", | ||
"type": "auth data", | ||
"logoFile": "logo.json", | ||
"defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*", | ||
"query": { | ||
"bool": { | ||
"filter": [ | ||
{ | ||
"term": { | ||
"event.category": "authentication" | ||
} | ||
} | ||
] | ||
} | ||
}, | ||
"jobs": [ | ||
{ | ||
"id": "auth_high_count_logon_events_for_a_source_ip", | ||
"file": "auth_high_count_logon_events_for_a_source_ip.json" | ||
}, | ||
{ | ||
"id": "auth_high_count_logon_fails", | ||
"file": "auth_high_count_logon_fails.json" | ||
}, | ||
{ | ||
"id": "auth_high_count_logon_events", | ||
"file": "auth_high_count_logon_events.json" | ||
}, | ||
{ | ||
"id": "auth_rare_hour_for_a_user", | ||
"file": "auth_rare_hour_for_a_user.json" | ||
}, | ||
{ | ||
"id": "auth_rare_source_ip_for_a_user", | ||
"file": "auth_rare_source_ip_for_a_user.json" | ||
}, | ||
{ | ||
"id": "auth_rare_user", | ||
"file": "auth_rare_user.json" | ||
} | ||
], | ||
"datafeeds": [ | ||
{ | ||
"id": "datafeed-auth_high_count_logon_events_for_a_source_ip", | ||
"file": "datafeed_auth_high_count_logon_events_for_a_source_ip.json", | ||
"job_id": "auth_high_count_logon_events_for_a_source_ip" | ||
}, | ||
{ | ||
"id": "datafeed-auth_high_count_logon_fails", | ||
"file": "datafeed_auth_high_count_logon_fails.json", | ||
"job_id": "auth_high_count_logon_fails" | ||
}, | ||
{ | ||
"id": "datafeed-auth_high_count_logon_events", | ||
"file": "datafeed_auth_high_count_logon_events.json", | ||
"job_id": "auth_high_count_logon_events" | ||
}, | ||
{ | ||
"id": "datafeed-auth_rare_hour_for_a_user", | ||
"file": "datafeed_auth_rare_hour_for_a_user.json", | ||
"job_id": "auth_rare_hour_for_a_user" | ||
}, | ||
{ | ||
"id": "datafeed-auth_rare_source_ip_for_a_user", | ||
"file": "datafeed_auth_rare_source_ip_for_a_user.json", | ||
"job_id": "auth_rare_source_ip_for_a_user" | ||
}, | ||
{ | ||
"id": "datafeed-auth_rare_user", | ||
"file": "datafeed_auth_rare_user.json", | ||
"job_id": "auth_rare_user" | ||
} | ||
] | ||
} |
29 changes: 29 additions & 0 deletions
29
.../server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for an unusually large spike in successful authentication events. This can be due to password spraying, user enumeration or brute force activity.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "high count of logon events", | ||
"function": "high_non_zero_count", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
34 changes: 34 additions & 0 deletions
34
...ata_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for an unusually large spike in successful authentication events events from a particular source IP address. This can be due to password spraying, user enumeration or brute force activity.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "high count of auth events for a source IP", | ||
"function": "high_non_zero_count", | ||
"by_field_name": "source.ip", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [ | ||
"source.ip", | ||
"winlog.event_data.LogonType", | ||
"user.name" | ||
] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
29 changes: 29 additions & 0 deletions
29
...l/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for an unusually large spike in authentication failure events. This can be due to password spraying, user enumeration or brute force activity and may be a precursor to account takeover or credentialed access.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "high count of logon fails", | ||
"function": "high_non_zero_count", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
33 changes: 33 additions & 0 deletions
33
.../ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for a user logging in at a time of day that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different time zones. In addition, unauthorized user activity often takes place during non-business hours.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "rare hour for a user", | ||
"function": "time_of_day", | ||
"by_field_name": "user.name", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [ | ||
"source.ip", | ||
"user.name" | ||
] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
34 changes: 34 additions & 0 deletions
34
...erver/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for a user logging in from an IP address that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different locations. An unusual source IP address for a username could also be due to lateral movement when a compromised account is used to pivot between hosts.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "rare source IP for a user", | ||
"function": "rare", | ||
"by_field_name": "source.ip", | ||
"partition_field_name": "user.name", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [ | ||
"source.ip", | ||
"user.name" | ||
] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
33 changes: 33 additions & 0 deletions
33
x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
{ | ||
"job_type": "anomaly_detector", | ||
"description": "Security: Authentication - looks for an unusual user name in the authentication logs. An unusual user name is one way of detecting credentialed access by means of a new or dormant user account. A user account that is normally inactive, because the user has left the organization, which becomes active, may be due to credentialed access using a compromised account password. Threat actors will sometimes also create new users as a means of persisting in a compromised web application.", | ||
"groups": [ | ||
"security", | ||
"authentication" | ||
], | ||
"analysis_config": { | ||
"bucket_span": "15m", | ||
"detectors": [ | ||
{ | ||
"detector_description": "rare user", | ||
"function": "rare", | ||
"by_field_name": "user.name", | ||
"detector_index": 0 | ||
} | ||
], | ||
"influencers": [ | ||
"source.ip", | ||
"user.name" | ||
] | ||
}, | ||
"allow_lazy_open": true, | ||
"analysis_limits": { | ||
"model_memory_limit": "128mb" | ||
}, | ||
"data_description": { | ||
"time_field": "@timestamp" | ||
}, | ||
"custom_settings": { | ||
"created_by": "ml-module-security-auth" | ||
} | ||
} |
26 changes: 26 additions & 0 deletions
26
...odels/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
{ | ||
"job_id": "auth_high_count_logon_events", | ||
"indices": [ | ||
"auditbeat-*", | ||
"logs-*", | ||
"filebeat-*", | ||
"winlogbeat-*" | ||
], | ||
"max_empty_searches": 10, | ||
"query": { | ||
"bool": { | ||
"filter": [ | ||
{ | ||
"term": { | ||
"event.category": "authentication" | ||
} | ||
}, | ||
{ | ||
"term": { | ||
"event.outcome": "success" | ||
} | ||
} | ||
] | ||
} | ||
} | ||
} |
26 changes: 26 additions & 0 deletions
26
...nizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
{ | ||
"job_id": "auth_high_count_logon_events_for_a_source_ip", | ||
"indices": [ | ||
"auditbeat-*", | ||
"logs-*", | ||
"filebeat-*", | ||
"winlogbeat-*" | ||
], | ||
"max_empty_searches": 10, | ||
"query": { | ||
"bool": { | ||
"filter": [ | ||
{ | ||
"term": { | ||
"event.category": "authentication" | ||
} | ||
}, | ||
{ | ||
"term": { | ||
"event.outcome": "success" | ||
} | ||
} | ||
] | ||
} | ||
} | ||
} |
26 changes: 26 additions & 0 deletions
26
...models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
{ | ||
"job_id": "auth_high_count_logon_fails", | ||
"indices": [ | ||
"auditbeat-*", | ||
"logs-*", | ||
"filebeat-*", | ||
"winlogbeat-*" | ||
], | ||
"max_empty_searches": 10, | ||
"query": { | ||
"bool": { | ||
"filter": [ | ||
{ | ||
"term": { | ||
"event.category": "authentication" | ||
} | ||
}, | ||
{ | ||
"term": { | ||
"event.outcome": "failure" | ||
} | ||
} | ||
] | ||
} | ||
} | ||
} |
26 changes: 26 additions & 0 deletions
26
...r/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
{ | ||
"job_id": "auth_rare_hour_for_a_user", | ||
"indices": [ | ||
"auditbeat-*", | ||
"logs-*", | ||
"filebeat-*", | ||
"winlogbeat-*" | ||
], | ||
"max_empty_searches": 10, | ||
"query": { | ||
"bool": { | ||
"filter": [ | ||
{ | ||
"term": { | ||
"event.category": "authentication" | ||
} | ||
}, | ||
{ | ||
"term": { | ||
"event.outcome": "success" | ||
} | ||
} | ||
] | ||
} | ||
} | ||
} |
Oops, something went wrong.