Skip to content

Commit

Permalink
Backport elastic#4609 to 5.x: New ML configurations for Filebeat Ngin…
Browse files Browse the repository at this point in the history
…x module

This backports elastic#4609 to the 5.x branch. The backport was done manually by copying the files,
because the folder structure for the dashboards changed.
  • Loading branch information
Tudor Golubenco committed Jul 5, 2017
1 parent 626a29f commit 2e7eaed
Show file tree
Hide file tree
Showing 21 changed files with 385 additions and 36 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ https://github.com/elastic/beats/compare/v5.4.1...master[Check the HEAD diff]

*Filebeat*

- Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506]
- Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] {pull}4609[4609]

*Heartbeat*

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"hits": 0,
"timeRestore": false,
"description": "",
"title": "ML Nginx Access Remote IP Count Explorer",
"uiStateJSON": "{\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
"panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Remote-IP-Timechart\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Map\",\"col\":7,\"row\":4},{\"size_x\":12,\"size_y\":9,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]",
"optionsJSON": "{\"darkTheme\":false}",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"hits": 0,
"timeRestore": false,
"description": "",
"title": "ML Nginx Access Remote IP URL Explorer",
"uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
"panelsJSON": "[{\"col\":1,\"id\":\"ML-Nginx-Access-Unique-Count-URL-Timechart\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Response-Code-Timechart\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ML-Nginx-Access-Top-Remote-IPs-Table\",\"panelIndex\":3,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Nginx-Access-Map\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Nginx-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]",
"optionsJSON": "{\"darkTheme\":false}",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"sort": [
"@timestamp",
"desc"
],
"hits": 0,
"description": "Filebeat Nginx Access Data",
"title": "ML Nginx Access Data",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"_exists_:nginx.access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
},
"columns": [
"_source"
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"autoPrecision\":true,\"field\":\"nginx.access.geoip.location\"},\"schema\":\"segment\",\"type\":\"geohash_grid\"}],\"listeners\":{},\"params\":{\"addTooltip\":true,\"heatBlur\":15,\"heatMaxZoom\":16,\"heatMinOpacity\":0.1,\"heatNormalizeData\":true,\"heatRadius\":25,\"isDesaturated\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[15,5],\"mapType\":\"Scaled Circle Markers\",\"mapZoom\":2,\"wms\":{\"enabled\":false,\"options\":{\"attribution\":\"Maps provided by USGS\",\"format\":\"image/png\",\"layers\":\"0\",\"styles\":\"\",\"transparent\":true,\"version\":\"1.3.0\"},\"url\":\"https://basemap.nationalmap.gov/arcgis/services/USGSTopo/MapServer/WMSServer\"}},\"title\":\"ML Nginx Access Map\",\"type\":\"tile_map\"}",
"description": "",
"title": "ML Nginx Access Map",
"uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"title\":\"ML Nginx Access Remote IP Timechart\",\"type\":\"area\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 5 minutes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"area\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"description": "",
"title": "ML Nginx Access Remote IP Timechart",
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"title\":\"ML Nginx Access Response Code Timechart\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"nginx.access.response_code\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"description": "",
"title": "ML Nginx Access Response Code Timechart",
"uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[]}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"title\":\"ML Nginx Access Top Remote IPs Table\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.remote_ip\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"description": "",
"title": "ML Nginx Access Top Remote IPs Table",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"title\":\"ML Nginx Access Top URLs Table\",\"type\":\"table\",\"params\":{\"perPage\":100,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"nginx.access.url\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"description": "",
"title": "ML Nginx Access Top URLs Table",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"visState": "{\"title\":\"ML Nginx Access Unique Count URL Timechart\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"@timestamp per day\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of nginx.access.url\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"line\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Unique count of nginx.access.url\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"nginx.access.url\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}",
"description": "",
"title": "ML Nginx Access Unique Count URL Timechart",
"uiStateJSON": "{}",
"version": 1,
"savedSearchId": "ML-Filebeat-Nginx-Access",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{}"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"job_id": "JOB_ID",
"query_delay": "60s",
"frequency": "60s",
"indexes": [
"filebeat-*"
],
"types": [
"_default_",
"log"
],
"query": {
"match_all": {
"boost": 1
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"interval": 900000,
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 0
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"job_id": "JOB_ID",
"query_delay": "60s",
"frequency": "60s",
"indexes": [
"filebeat-*"
],
"types": [
"_default_",
"log"
],
"query": {
"match_all": {
"boost": 1
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"job_id": "JOB_ID",
"query_delay": "60s",
"frequency": "60s",
"indexes": [
"filebeat-*"
],
"types": [
"_default_",
"log"
],
"query": {
"match_all": {
"boost": 1
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,32 +13,5 @@
"match_all": {
"boost": 1
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"interval": 3600000,
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 0
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
},
"nginx.access.response_code": {
"terms": {
"field": "nginx.access.response_code",
"size": 10000
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{
"job_id": "JOB_ID",
"query_delay": "60s",
"frequency": "60s",
"indexes": [
"filebeat-*"
],
"types": [
"_default_",
"log"
],
"query": {
"match_all": {
"boost": 1
}
},
"aggregations": {
"buckets": {
"date_histogram": {
"field": "@timestamp",
"interval": 900000,
"offset": 0,
"order": {
"_key": "asc"
},
"keyed": false,
"min_doc_count": 0
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
},
"dc_remote_ips": {
"cardinality": {
"field": "nginx.access.remote_ip"
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{
"description": "Nginx Access Logs: Detect low request rate",
"analysis_config" : {
"bucket_span": "15m",
"summary_count_field_name": "doc_count",
"detectors": [
{
"detector_description": "nginx_access_low_request_rate",
"function": "low_count",
"detector_rules": []
}
],
"influencers": []
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"model_plot_config": {
"enabled": true
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Raw Data",
"url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!(),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"description": "Nginx Access Logs: Detect unusual remote_ips - high request rates",
"analysis_config" : {
"bucket_span": "1h",
"detectors": [
{
"detector_description": "nginx_access_remote_ip_high_count",
"function": "high_count",
"over_field_name": "nginx.access.remote_ip",
"detector_rules": []
}
],
"influencers": [
"nginx.access.remote_ip"
]
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Count Explorer",
"url_value": "kibana#/dashboard/ML-Nginx-Access-Remote-IP-Count-Explorer?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),options:(darkTheme:!f),panels:!((col:1,id:ML-Nginx-Access-Remote-IP-Timechart,panelIndex:1,row:1,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Response-Code-Timechart,panelIndex:2,row:1,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-Remote-IPs-Table,panelIndex:3,row:4,size_x:6,size_y:3,type:visualization),(col:7,id:ML-Nginx-Access-Map,panelIndex:4,row:4,size_x:6,size_y:3,type:visualization),(col:1,id:ML-Nginx-Access-Top-URLs-Table,panelIndex:5,row:7,size_x:12,size_y:9,type:visualization)),query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),timeRestore:!f,title:\u0027ML%20Nginx%20Access%20Remote%20IP%20Count%20Explorer\u0027,uiState:(P-3:(vis:(params:(sort:(columnIndex:!n,direction:!n)))),P-5:(vis:(params:(sort:(columnIndex:!n,direction:!n))))),viewMode:view)"
},
{
"url_name": "Raw Data",
"url_value": "kibana#/discover/ML-Filebeat-Nginx-Access?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027filebeat-*\u0027,key:nginx.access.remote_ip,negate:!f,type:phrase,value:\u0027$nginx.access.remote_ip$\u0027),query:(match:(nginx.access.remote_ip:(query:\u0027$nginx.access.remote_ip$\u0027,type:phrase))))),index:\u0027filebeat-*\u0027,interval:auto,query:(query_string:(analyze_wildcard:!t,query:\u0027*\u0027)),sort:!(\u0027@timestamp\u0027,desc))"
}
]
}
}
Loading

0 comments on commit 2e7eaed

Please sign in to comment.