Skip to content

Commit

Permalink
[ML] Adds ML modules for Metrics UI Integration (elastic#76460)
Browse files Browse the repository at this point in the history
* adds metrics ml integration

* renames jobs, updates datafeeds

* adds allow_no_indices: true for datafeeds

* updates module ids in manifest

* adds custom urls

* adds module and individual job descriptions

* removes model plots

* updates terms agg sizes

* updates chunking config

* removes query and default index pattern from manifest, updates descriptions

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
  • Loading branch information
2 people authored and blaklaybul committed Sep 17, 2020
1 parent 4c734e4 commit 332b56c
Show file tree
Hide file tree
Showing 17 changed files with 540 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"icon": "metricsApp"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"id": "metrics_ui_hosts",
"title": "Metrics Hosts",
"description": "Detect anomalous memory and network behavior on hosts.",
"type": "Metricbeat Data",
"logoFile": "logo.json",
"jobs": [
{
"id": "hosts_memory_usage",
"file": "hosts_memory_usage.json"
},
{
"id": "hosts_network_in",
"file": "hosts_network_in.json"
},
{
"id": "hosts_network_out",
"file": "hosts_network_out.json"
}
],
"datafeeds": [
{
"id": "datafeed-hosts_memory_usage",
"file": "datafeed_hosts_memory_usage.json",
"job_id": "hosts_memory_usage"
},
{
"id": "datafeed-hosts_network_in",
"file": "datafeed_hosts_network_in.json",
"job_id": "hosts_network_in"
},
{
"id": "datafeed-hosts_network_out",
"file": "datafeed_hosts_network_out.json",
"job_id": "hosts_network_out"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"must": [
{"exists": {"field": "system.memory"}}
]
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"must": [
{"exists": {"field": "system.network"}}
]
}
},
"chunking_config": {
"mode": "manual",
"time_span": "900s"
},
"aggregations": {
"host.name": {"terms": {"field": "host.name", "size": 100},
"aggregations": {
"buckets": {
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
"aggregations": {
"@timestamp": {"max": {"field": "@timestamp"}},
"bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
"bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
"positive_only":{
"bucket_script": {
"buckets_path": {"in_derivative": "bytes_in_derivative.value"},
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
}
}
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"must": [
{"exists": {"field": "system.network"}}
]
}
},
"chunking_config": {
"mode": "manual",
"time_span": "900s"
},
"aggregations": {
"host.name": {"terms": {"field": "host.name", "size": 100},
"aggregations": {
"buckets": {
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
"aggregations": {
"@timestamp": {"max": {"field": "@timestamp"}},
"bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
"bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
"positive_only":{
"bucket_script": {
"buckets_path": {"out_derivative": "bytes_out_derivative.value"},
"script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
}
}
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
{
"job_type": "anomaly_detector",
"groups": [
"hosts",
"metrics"
],
"description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "max('system.memory.actual.used.pct')",
"function": "max",
"field_name": "system.memory.actual.used.pct",
"custom_rules": [
{
"actions": [
"skip_result"
],
"conditions": [
{
"applies_to": "actual",
"operator": "lt",
"value": 0.1
}
]
}
]
}
],
"influencers": [
"host.name"
]
},
"data_description": {
"time_field": "@timestamp"
},
"analysis_limits": {
"model_memory_limit": "64mb"
},
"custom_settings": {
"created_by": "ml-module-metrics-ui-hosts",
"custom_urls": [
{
"url_name": "Host Metrics",
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"job_type": "anomaly_detector",
"description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
"groups": [
"hosts",
"metrics"
],
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "max(bytes_in_derivative)",
"function": "max",
"field_name": "bytes_in_derivative"
}
],
"influencers": [
"host.name"
],
"summary_count_field_name": "doc_count"
},
"data_description": {
"time_field": "@timestamp"
},
"analysis_limits": {
"model_memory_limit": "32mb"
},
"custom_settings": {
"created_by": "ml-module-metrics-ui-hosts",
"custom_urls": [
{
"url_name": "Host Metrics",
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
{
"job_type": "anomaly_detector",
"description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
"groups": [
"hosts",
"metrics"
],
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "max(bytes_out_derivative)",
"function": "max",
"field_name": "bytes_out_derivative"
}
],
"influencers": [
"host.name"
],
"summary_count_field_name": "doc_count"
},
"data_description": {
"time_field": "@timestamp"
},
"analysis_limits": {
"model_memory_limit": "32mb"
},
"custom_settings": {
"created_by": "ml-module-metrics-ui-hosts",
"custom_urls": [
{
"url_name": "Host Metrics",
"url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"icon": "metricsApp"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"id": "metrics_ui_k8s",
"title": "Metrics Kubernetes",
"description": "Detect anomalous memory and network behavior on Kubernetes pods.",
"type": "Metricbeat Data",
"logoFile": "logo.json",
"jobs": [
{
"id": "k8s_memory_usage",
"file": "k8s_memory_usage.json"
},
{
"id": "k8s_network_in",
"file": "k8s_network_in.json"
},
{
"id": "k8s_network_out",
"file": "k8s_network_out.json"
}
],
"datafeeds": [
{
"id": "datafeed-k8s_memory_usage",
"file": "datafeed_k8s_memory_usage.json",
"job_id": "k8s_memory_usage"
},
{
"id": "datafeed-k8s_network_in",
"file": "datafeed_k8s_network_in.json",
"job_id": "k8s_network_in"
},
{
"id": "datafeed-k8s_network_out",
"file": "datafeed_k8s_network_out.json",
"job_id": "k8s_network_out"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"must": [
{"exists": {"field": "kubernetes.pod.uid"}},
{"exists": {"field": "kubernetes.pod.memory"}}
]
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"job_id": "JOB_ID",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"must": [
{"exists": {"field": "kubernetes.pod.network"}}
]
}
},
"chunking_config": {
"mode": "manual",
"time_span": "900s"
},
"aggregations": {
"kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25},
"aggregations": {
"kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100},
"aggregations": {
"buckets": {
"date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
"aggregations": {
"@timestamp": {"max": {"field": "@timestamp"}},
"bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}},
"bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
"positive_only":{
"bucket_script": {
"buckets_path": {"in_derivative": "bytes_in_derivative.value"},
"script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
}
}
}
}
}
}
}
}
}
}
Loading

0 comments on commit 332b56c

Please sign in to comment.