[ML] Adds siem_cloudtrail Module

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/logo.json

@@ -0,0 +1,3 @@
+{
+    "icon": "logoSecurity"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/manifest.json

@@ -0,0 +1,64 @@
+{
+    "id": "siem_cloudtrail",
+    "title": "SIEM Cloudtrail",
+    "description": "Detect suspicious activity recorded in your cloudtrail logs.",
+    "type": "Filebeat data",
+    "logoFile": "logo.json",
+    "defaultIndexPattern": "filebeat-*",
+    "query": {
+      "bool": {
+        "filter": [
+          {"term": {"event.dataset": "aws.cloudtrail"}}
+        ]
+      }
+    },
+    "jobs": [
+      {
+        "id": "rare_method_for_a_city",
+        "file": "rare_method_for_a_city.json"
+      },
+      {
+        "id": "rare_method_for_a_country",
+        "file": "rare_method_for_a_country.json"
+      },
+      {
+        "id": "rare_method_for_a_username",
+        "file": "rare_method_for_a_username.json"
+      },
+      {
+        "id": "high_distinct_count_error_message",
+        "file": "high_distinct_count_error_message.json"
+      },
+      {
+        "id": "rare_error_code",
+        "file": "rare_error_code.json"
+      }
+    ],
+    "datafeeds": [
+      {
+        "id": "datafeed-rare_method_for_a_city",
+        "file": "datafeed_rare_method_for_a_city.json",
+        "job_id": "rare_method_for_a_city"
+      },
+      {
+        "id": "datafeed-rare_method_for_a_country",
+        "file": "datafeed_rare_method_for_a_country.json",
+        "job_id": "rare_method_for_a_country"
+      },
+      {
+        "id": "datafeed-rare_method_for_a_username",
+        "file": "datafeed_rare_method_for_a_username.json",
+        "job_id": "rare_method_for_a_username"
+      },
+      {
+        "id": "datafeed-high_distinct_count_error_message",
+        "file": "datafeed_high_distinct_count_error_message.json",
+        "job_id": "high_distinct_count_error_message"
+      },
+      {
+        "id": "datafeed-rare_error_code",
+        "file": "datafeed_rare_error_code.json",
+        "job_id": "rare_error_code"
+      }
+    ]
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_high_distinct_count_error_message.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [
+                {"term": {"event.dataset": "aws.cloudtrail"}},
+                {"term": {"event.module": "aws"}},
+                {"exists": {"field": "aws.cloudtrail.error_message"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_error_code.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [
+                {"term": {"event.dataset": "aws.cloudtrail"}},
+                {"term": {"event.module": "aws"}},
+                {"exists": {"field": "aws.cloudtrail.error_code"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_city.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [
+                {"term": {"event.dataset": "aws.cloudtrail"}},
+                {"term": {"event.module": "aws"}},
+                {"exists": {"field": "source.geo.city_name"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_country.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [
+                {"term": {"event.dataset": "aws.cloudtrail"}},
+                {"term": {"event.module": "aws"}},
+                {"exists": {"field": "source.geo.country_iso_code"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/datafeed_rare_method_for_a_username.json

@@ -0,0 +1,16 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [
+                {"term": {"event.dataset": "aws.cloudtrail"}},
+                {"term": {"event.module": "aws"}},
+                {"exists": {"field": "user.name"}}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/high_distinct_count_error_message.json

@@ -0,0 +1,33 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Looks for a spike in the rate of an error message which may simply indicate an impending service failure but these can also be byproducts of attempted or successful persistence, privilege escalation, defense evasion, discovery, lateral movement, or collection activity by a threat actor.",
+    "groups": [
+      "siem",
+      "cloudtrail"
+    ],
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+          "detector_description": "high_distinct_count(\"aws.cloudtrail.error_message\")",
+          "function": "high_distinct_count",
+          "field_name": "aws.cloudtrail.error_message"
+        }
+      ],
+      "influencers": [
+        "aws.cloudtrail.user_identity.arn",
+        "source.ip",
+        "source.geo.city_name"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "16mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-cloudtrail"
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_error_code.json

@@ -0,0 +1,33 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Looks for unsual errors. Rare and unusual errors may simply indicate an impending service failure but they can also be byproducts of attempted or successful persistence, privilege escalation, defense evasion, discovery, lateral movement, or collection activity by a threat actor.",
+    "groups": [
+      "siem",
+      "cloudtrail"
+    ],
+    "analysis_config": {
+      "bucket_span": "60m",
+      "detectors": [
+        {
+          "detector_description": "rare by \"aws.cloudtrail.error_code\"",
+          "function": "rare",
+          "by_field_name": "aws.cloudtrail.error_code"
+        }
+      ],
+      "influencers": [
+        "aws.cloudtrail.user_identity.arn",
+        "source.ip",
+        "source.geo.city_name"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "16mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-cloudtrail"
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_city.json

@@ -0,0 +1,34 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a geolocation (city) that is unusual. This can be the result of compromised credentials or keys.",
+    "groups": [
+      "siem",
+      "cloudtrail"
+    ],
+    "analysis_config": {
+      "bucket_span": "60m",
+      "detectors": [
+        {
+          "detector_description": "rare by \"event.action\" partition by \"source.geo.city_name\"",
+          "function": "rare",
+          "by_field_name": "event.action",
+          "partition_field_name": "source.geo.city_name"
+        }
+      ],
+      "influencers": [
+        "aws.cloudtrail.user_identity.arn",
+        "source.ip",
+        "source.geo.city_name"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-cloudtrail"
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_country.json

@@ -0,0 +1,34 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a geolocation (country) that is unusual. This can be the result of compromised credentials or keys.",
+    "groups": [
+      "siem",
+      "cloudtrail"
+    ],
+    "analysis_config": {
+      "bucket_span": "60m",
+      "detectors": [
+        {
+          "detector_description": "rare by \"event.action\" partition by \"source.geo.country_iso_code\"",
+          "function": "rare",
+          "by_field_name": "event.action",
+          "partition_field_name": "source.geo.country_iso_code"
+        }
+      ],
+      "influencers": [
+        "aws.cloudtrail.user_identity.arn",
+        "source.ip",
+        "source.geo.country_iso_code"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-cloudtrail"
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_cloudtrail/ml/rare_method_for_a_username.json

@@ -0,0 +1,34 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Looks for AWS API calls that, while not inherently suspicious or abnormal, are sourcing from a user context that does not normally call the method. This can be the result of compromised credentials or keys as someone uses a valid account to persist, move laterally, or exfil data.",
+    "groups": [
+      "siem",
+      "cloudtrail"
+    ],
+    "analysis_config": {
+      "bucket_span": "60m",
+      "detectors": [
+        {
+          "detector_description": "rare by \"event.action\" partition by \"user.name\"",
+          "function": "rare",
+          "by_field_name": "event.action",
+          "partition_field_name": "user.name"
+        }
+      ],
+      "influencers": [
+        "user.name",
+        "source.ip",
+        "source.geo.city_name"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "128mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-cloudtrail"
+    }
+  }

x-pack/test/api_integration/apis/ml/modules/get_module.ts

@@ -25,6 +25,7 @@ const moduleIds = [
   'sample_data_weblogs',
   'siem_auditbeat',
   'siem_auditbeat_auth',
+  'siem_cloudtrail',
   'siem_packetbeat',
   'siem_winlogbeat',
   'siem_winlogbeat_auth',