[ML] Adds Authentication module with six ML jobs for ECS data (Auditb…

…eat, Winlogbeat, Filebeat and Logs) (elastic#101840)

* Adding Security Authentication jobs in 7.14

* Renamed some jobs

* Changing memory limits and linting change

* Linting fix

* Changed the order

* Adding module to ml_modules.tsx

* Update recognize_module.ts

this test modules uses older Auditbeat data which predates the event.category field so the test has to be skipped per https://elastic.zoom.us/j/93000943632?pwd=TmpvNWhtYUNzMUc0c0N6Tlc2QlVPZz09

* Update recognize_module.ts

needs to be a single line

* Update recognize_module.ts

Some linters want spaces and some linters want no spaces. This linter wants spaces.

* descriptions

added description text

* Update auth_rare_hour_for_a_user.json

removed a wayward newline char

* Minor nitpicking

* memory limits

raised memory limits to 128mb which is larger than the highest observed peak model bytes for the most memory hungry jobs in this event class.

Co-authored-by: Craig <mailredirector36@gmail.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "logoSecurity"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/manifest.json

@@ -0,0 +1,77 @@
+{
+  "id": "security_auth",
+  "title": "Security: Authentication",
+  "description": "Detect anomalous activity in your ECS-compatible authentication logs.",
+  "type": "auth data",
+  "logoFile": "logo.json",
+  "defaultIndexPattern": "auditbeat-*,logs-*,filebeat-*,winlogbeat-*",
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "authentication"
+          }
+        }
+      ]
+    }
+  },
+  "jobs": [
+    {
+      "id": "auth_high_count_logon_events_for_a_source_ip",
+      "file": "auth_high_count_logon_events_for_a_source_ip.json"
+    },
+    {
+      "id": "auth_high_count_logon_fails",
+      "file": "auth_high_count_logon_fails.json"
+    },
+    {
+      "id": "auth_high_count_logon_events",
+      "file": "auth_high_count_logon_events.json"
+    },
+    {
+      "id": "auth_rare_hour_for_a_user",
+      "file": "auth_rare_hour_for_a_user.json"
+    },
+    {
+      "id": "auth_rare_source_ip_for_a_user",
+      "file": "auth_rare_source_ip_for_a_user.json"
+    },
+    {
+      "id": "auth_rare_user",
+      "file": "auth_rare_user.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed-auth_high_count_logon_events_for_a_source_ip",
+      "file": "datafeed_auth_high_count_logon_events_for_a_source_ip.json",
+      "job_id": "auth_high_count_logon_events_for_a_source_ip"
+    },
+    {
+      "id": "datafeed-auth_high_count_logon_fails",
+      "file": "datafeed_auth_high_count_logon_fails.json",
+      "job_id": "auth_high_count_logon_fails"
+    },
+    {
+      "id": "datafeed-auth_high_count_logon_events",
+      "file": "datafeed_auth_high_count_logon_events.json",
+      "job_id": "auth_high_count_logon_events"
+    },
+    {
+      "id": "datafeed-auth_rare_hour_for_a_user",
+      "file": "datafeed_auth_rare_hour_for_a_user.json",
+      "job_id": "auth_rare_hour_for_a_user"
+    },
+    {
+      "id": "datafeed-auth_rare_source_ip_for_a_user",
+      "file": "datafeed_auth_rare_source_ip_for_a_user.json",
+      "job_id": "auth_rare_source_ip_for_a_user"
+    },
+    {
+      "id": "datafeed-auth_rare_user",
+      "file": "datafeed_auth_rare_user.json",
+      "job_id": "auth_rare_user"
+    }
+  ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events.json

@@ -0,0 +1,29 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for an unusually large spike in successful authentication events. This can be due to password spraying, user enumeration or brute force activity.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high count of logon events",
+        "function": "high_non_zero_count",
+        "detector_index": 0
+      }
+    ],
+    "influencers": []
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_events_for_a_source_ip.json

@@ -0,0 +1,34 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for an unusually large spike in successful authentication events events from a particular source IP address. This can be due to password spraying, user enumeration or brute force activity.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high count of auth events for a source IP",
+        "function": "high_non_zero_count",
+        "by_field_name": "source.ip",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "source.ip",
+      "winlog.event_data.LogonType",
+      "user.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_high_count_logon_fails.json

@@ -0,0 +1,29 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for an unusually large spike in authentication failure events. This can be due to password spraying, user enumeration or brute force activity and may be a precursor to account takeover or credentialed access.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high count of logon fails",
+        "function": "high_non_zero_count",
+        "detector_index": 0
+      }
+    ],
+    "influencers": []
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_hour_for_a_user.json

@@ -0,0 +1,33 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for a user logging in at a time of day that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different time zones. In addition, unauthorized user activity often takes place during non-business hours.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "rare hour for a user",
+        "function": "time_of_day",
+        "by_field_name": "user.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "source.ip",
+      "user.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_source_ip_for_a_user.json

@@ -0,0 +1,34 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for a user logging in from an IP address that is unusual for the user. This can be due to credentialed access via a compromised account when the user and the threat actor are in different locations. An unusual source IP address for a username could also be due to lateral movement when a compromised account is used to pivot between hosts.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "rare source IP for a user",
+        "function": "rare",
+        "by_field_name": "source.ip",
+        "partition_field_name": "user.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "source.ip",
+      "user.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/auth_rare_user.json

@@ -0,0 +1,33 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Authentication - looks for an unusual user name in the authentication logs. An unusual user name is one way of detecting credentialed access by means of a new or dormant user account. A user account that is normally inactive, because the user has left the organization, which becomes active, may be due to credentialed access using a compromised account password. Threat actors will sometimes also create new users as a means of persisting in a compromised web application.",
+  "groups": [
+    "security",
+    "authentication"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "rare user",
+        "function": "rare",
+        "by_field_name": "user.name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "source.ip",
+      "user.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-auth"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events.json

@@ -0,0 +1,26 @@
+{
+  "job_id": "auth_high_count_logon_events",
+  "indices": [
+    "auditbeat-*",
+    "logs-*",
+    "filebeat-*",
+    "winlogbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "authentication"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_events_for_a_source_ip.json

@@ -0,0 +1,26 @@
+{
+  "job_id": "auth_high_count_logon_events_for_a_source_ip",
+  "indices": [
+    "auditbeat-*",
+    "logs-*",
+    "filebeat-*",
+    "winlogbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "authentication"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_high_count_logon_fails.json

@@ -0,0 +1,26 @@
+{
+  "job_id": "auth_high_count_logon_fails",
+  "indices": [
+    "auditbeat-*",
+    "logs-*",
+    "filebeat-*",
+    "winlogbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "authentication"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "failure"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_auth/ml/datafeed_auth_rare_hour_for_a_user.json

@@ -0,0 +1,26 @@
+{
+  "job_id": "auth_rare_hour_for_a_user",
+  "indices": [
+    "auditbeat-*",
+    "logs-*",
+    "filebeat-*",
+    "winlogbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "authentication"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "success"
+          }
+        }
+      ]
+    }
+  }
+}