forked from elastic/kibana
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] Adds ML modules for Metrics UI Integration (elastic#76460)
* adds metrics ml integration * renames jobs, updates datafeeds * adds allow_no_indices: true for datafeeds * updates module ids in manifest * adds custom urls * adds module and individual job descriptions * removes model plots * updates terms agg sizes * updates chunking config * removes query and default index pattern from manifest, updates descriptions Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
- Loading branch information
1 parent
4c734e4
commit 332b56c
Showing
17 changed files
with
540 additions
and
0 deletions.
There are no files selected for viewing
3 changes: 3 additions & 0 deletions
3
x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| { | ||
| "icon": "metricsApp" | ||
| } |
38 changes: 38 additions & 0 deletions
38
x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| { | ||
| "id": "metrics_ui_hosts", | ||
| "title": "Metrics Hosts", | ||
| "description": "Detect anomalous memory and network behavior on hosts.", | ||
| "type": "Metricbeat Data", | ||
| "logoFile": "logo.json", | ||
| "jobs": [ | ||
| { | ||
| "id": "hosts_memory_usage", | ||
| "file": "hosts_memory_usage.json" | ||
| }, | ||
| { | ||
| "id": "hosts_network_in", | ||
| "file": "hosts_network_in.json" | ||
| }, | ||
| { | ||
| "id": "hosts_network_out", | ||
| "file": "hosts_network_out.json" | ||
| } | ||
| ], | ||
| "datafeeds": [ | ||
| { | ||
| "id": "datafeed-hosts_memory_usage", | ||
| "file": "datafeed_hosts_memory_usage.json", | ||
| "job_id": "hosts_memory_usage" | ||
| }, | ||
| { | ||
| "id": "datafeed-hosts_network_in", | ||
| "file": "datafeed_hosts_network_in.json", | ||
| "job_id": "hosts_network_in" | ||
| }, | ||
| { | ||
| "id": "datafeed-hosts_network_out", | ||
| "file": "datafeed_hosts_network_out.json", | ||
| "job_id": "hosts_network_out" | ||
| } | ||
| ] | ||
| } |
16 changes: 16 additions & 0 deletions
16
...erver/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| { | ||
| "job_id": "JOB_ID", | ||
| "indices": [ | ||
| "INDEX_PATTERN_NAME" | ||
| ], | ||
| "indices_options": { | ||
| "allow_no_indices": true | ||
| }, | ||
| "query": { | ||
| "bool": { | ||
| "must": [ | ||
| {"exists": {"field": "system.memory"}} | ||
| ] | ||
| } | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
.../server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| { | ||
| "job_id": "JOB_ID", | ||
| "indices": [ | ||
| "INDEX_PATTERN_NAME" | ||
| ], | ||
| "indices_options": { | ||
| "allow_no_indices": true | ||
| }, | ||
| "query": { | ||
| "bool": { | ||
| "must": [ | ||
| {"exists": {"field": "system.network"}} | ||
| ] | ||
| } | ||
| }, | ||
| "chunking_config": { | ||
| "mode": "manual", | ||
| "time_span": "900s" | ||
| }, | ||
| "aggregations": { | ||
| "host.name": {"terms": {"field": "host.name", "size": 100}, | ||
| "aggregations": { | ||
| "buckets": { | ||
| "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, | ||
| "aggregations": { | ||
| "@timestamp": {"max": {"field": "@timestamp"}}, | ||
| "bytes_in_max": {"max": {"field": "system.network.in.bytes"}}, | ||
| "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, | ||
| "positive_only":{ | ||
| "bucket_script": { | ||
| "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, | ||
| "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
...server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| { | ||
| "job_id": "JOB_ID", | ||
| "indices": [ | ||
| "INDEX_PATTERN_NAME" | ||
| ], | ||
| "indices_options": { | ||
| "allow_no_indices": true | ||
| }, | ||
| "query": { | ||
| "bool": { | ||
| "must": [ | ||
| {"exists": {"field": "system.network"}} | ||
| ] | ||
| } | ||
| }, | ||
| "chunking_config": { | ||
| "mode": "manual", | ||
| "time_span": "900s" | ||
| }, | ||
| "aggregations": { | ||
| "host.name": {"terms": {"field": "host.name", "size": 100}, | ||
| "aggregations": { | ||
| "buckets": { | ||
| "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, | ||
| "aggregations": { | ||
| "@timestamp": {"max": {"field": "@timestamp"}}, | ||
| "bytes_out_max": {"max": {"field": "system.network.out.bytes"}}, | ||
| "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}}, | ||
| "positive_only":{ | ||
| "bucket_script": { | ||
| "buckets_path": {"out_derivative": "bytes_out_derivative.value"}, | ||
| "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
50 changes: 50 additions & 0 deletions
50
...gins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| { | ||
| "job_type": "anomaly_detector", | ||
| "groups": [ | ||
| "hosts", | ||
| "metrics" | ||
| ], | ||
| "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.", | ||
| "analysis_config": { | ||
| "bucket_span": "15m", | ||
| "detectors": [ | ||
| { | ||
| "detector_description": "max('system.memory.actual.used.pct')", | ||
| "function": "max", | ||
| "field_name": "system.memory.actual.used.pct", | ||
| "custom_rules": [ | ||
| { | ||
| "actions": [ | ||
| "skip_result" | ||
| ], | ||
| "conditions": [ | ||
| { | ||
| "applies_to": "actual", | ||
| "operator": "lt", | ||
| "value": 0.1 | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| ], | ||
| "influencers": [ | ||
| "host.name" | ||
| ] | ||
| }, | ||
| "data_description": { | ||
| "time_field": "@timestamp" | ||
| }, | ||
| "analysis_limits": { | ||
| "model_memory_limit": "64mb" | ||
| }, | ||
| "custom_settings": { | ||
| "created_by": "ml-module-metrics-ui-hosts", | ||
| "custom_urls": [ | ||
| { | ||
| "url_name": "Host Metrics", | ||
| "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" | ||
| } | ||
| ] | ||
| } | ||
| } |
37 changes: 37 additions & 0 deletions
37
...lugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| { | ||
| "job_type": "anomaly_detector", | ||
| "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.", | ||
| "groups": [ | ||
| "hosts", | ||
| "metrics" | ||
| ], | ||
| "analysis_config": { | ||
| "bucket_span": "15m", | ||
| "detectors": [ | ||
| { | ||
| "detector_description": "max(bytes_in_derivative)", | ||
| "function": "max", | ||
| "field_name": "bytes_in_derivative" | ||
| } | ||
| ], | ||
| "influencers": [ | ||
| "host.name" | ||
| ], | ||
| "summary_count_field_name": "doc_count" | ||
| }, | ||
| "data_description": { | ||
| "time_field": "@timestamp" | ||
| }, | ||
| "analysis_limits": { | ||
| "model_memory_limit": "32mb" | ||
| }, | ||
| "custom_settings": { | ||
| "created_by": "ml-module-metrics-ui-hosts", | ||
| "custom_urls": [ | ||
| { | ||
| "url_name": "Host Metrics", | ||
| "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" | ||
| } | ||
| ] | ||
| } | ||
| } |
37 changes: 37 additions & 0 deletions
37
...ugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| { | ||
| "job_type": "anomaly_detector", | ||
| "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.", | ||
| "groups": [ | ||
| "hosts", | ||
| "metrics" | ||
| ], | ||
| "analysis_config": { | ||
| "bucket_span": "15m", | ||
| "detectors": [ | ||
| { | ||
| "detector_description": "max(bytes_out_derivative)", | ||
| "function": "max", | ||
| "field_name": "bytes_out_derivative" | ||
| } | ||
| ], | ||
| "influencers": [ | ||
| "host.name" | ||
| ], | ||
| "summary_count_field_name": "doc_count" | ||
| }, | ||
| "data_description": { | ||
| "time_field": "@timestamp" | ||
| }, | ||
| "analysis_limits": { | ||
| "model_memory_limit": "32mb" | ||
| }, | ||
| "custom_settings": { | ||
| "created_by": "ml-module-metrics-ui-hosts", | ||
| "custom_urls": [ | ||
| { | ||
| "url_name": "Host Metrics", | ||
| "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))" | ||
| } | ||
| ] | ||
| } | ||
| } |
3 changes: 3 additions & 0 deletions
3
x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| { | ||
| "icon": "metricsApp" | ||
| } |
38 changes: 38 additions & 0 deletions
38
x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| { | ||
| "id": "metrics_ui_k8s", | ||
| "title": "Metrics Kubernetes", | ||
| "description": "Detect anomalous memory and network behavior on Kubernetes pods.", | ||
| "type": "Metricbeat Data", | ||
| "logoFile": "logo.json", | ||
| "jobs": [ | ||
| { | ||
| "id": "k8s_memory_usage", | ||
| "file": "k8s_memory_usage.json" | ||
| }, | ||
| { | ||
| "id": "k8s_network_in", | ||
| "file": "k8s_network_in.json" | ||
| }, | ||
| { | ||
| "id": "k8s_network_out", | ||
| "file": "k8s_network_out.json" | ||
| } | ||
| ], | ||
| "datafeeds": [ | ||
| { | ||
| "id": "datafeed-k8s_memory_usage", | ||
| "file": "datafeed_k8s_memory_usage.json", | ||
| "job_id": "k8s_memory_usage" | ||
| }, | ||
| { | ||
| "id": "datafeed-k8s_network_in", | ||
| "file": "datafeed_k8s_network_in.json", | ||
| "job_id": "k8s_network_in" | ||
| }, | ||
| { | ||
| "id": "datafeed-k8s_network_out", | ||
| "file": "datafeed_k8s_network_out.json", | ||
| "job_id": "k8s_network_out" | ||
| } | ||
| ] | ||
| } |
17 changes: 17 additions & 0 deletions
17
...ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| { | ||
| "job_id": "JOB_ID", | ||
| "indices": [ | ||
| "INDEX_PATTERN_NAME" | ||
| ], | ||
| "indices_options": { | ||
| "allow_no_indices": true | ||
| }, | ||
| "query": { | ||
| "bool": { | ||
| "must": [ | ||
| {"exists": {"field": "kubernetes.pod.uid"}}, | ||
| {"exists": {"field": "kubernetes.pod.memory"}} | ||
| ] | ||
| } | ||
| } | ||
| } |
44 changes: 44 additions & 0 deletions
44
...s/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| { | ||
| "job_id": "JOB_ID", | ||
| "indices": [ | ||
| "INDEX_PATTERN_NAME" | ||
| ], | ||
| "indices_options": { | ||
| "allow_no_indices": true | ||
| }, | ||
| "query": { | ||
| "bool": { | ||
| "must": [ | ||
| {"exists": {"field": "kubernetes.pod.network"}} | ||
| ] | ||
| } | ||
| }, | ||
| "chunking_config": { | ||
| "mode": "manual", | ||
| "time_span": "900s" | ||
| }, | ||
| "aggregations": { | ||
| "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25}, | ||
| "aggregations": { | ||
| "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100}, | ||
| "aggregations": { | ||
| "buckets": { | ||
| "date_histogram": {"field": "@timestamp","fixed_interval": "5m"}, | ||
| "aggregations": { | ||
| "@timestamp": {"max": {"field": "@timestamp"}}, | ||
| "bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}}, | ||
| "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}}, | ||
| "positive_only":{ | ||
| "bucket_script": { | ||
| "buckets_path": {"in_derivative": "bytes_in_derivative.value"}, | ||
| "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.