Add Elasticsearch 5.x output

Godeps

@@ -29,6 +29,7 @@ github.com/influxdata/config b79f6829346b8d6e78ba73544b1e1038f1f1c9da
 github.com/influxdata/influxdb fc57c0f7c635df3873f3d64f0ed2100ddc94d5ae
 github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
 github.com/influxdata/wlog 7c63b0a71ef8300adc255344d275e10e5c3a71ec
+github.com/jackc/pgx bb73d8427902891bbad7b949b9c60b32949d935f
 github.com/kardianos/osext 29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc
 github.com/kardianos/service 5e335590050d6d00f3aa270217d288dda1c94d0a
 github.com/kballard/go-shellquote d8ec1a69a250a17bb0e419c386eac1f3711dc142
@@ -61,5 +62,5 @@ golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34
 gopkg.in/dancannon/gorethink.v1 7d1af5be49cb5ecc7b177bf387d232050299d6ef
 gopkg.in/fatih/pool.v2 cba550ebf9bce999a02e963296d4bc7a486cb715
 gopkg.in/mgo.v2 d90005c5262a3463800497ea5a89aed5fe22c886
+gopkg.in/olivere/elastic.v5 ee3ebceab960cf68ab9a89ee6d78c031ef5b4a4e
 gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4
-github.com/jackc/pgx bb73d8427902891bbad7b949b9c60b32949d935f

Makefile

@@ -51,6 +51,7 @@ docker-run:
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
+	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
 	docker run --name mysql -p "3306:3306" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql
 	docker run --name memcached -p "11211:11211" -d memcached
 	docker run --name postgres -p "5432:5432" -d postgres
@@ -69,15 +70,16 @@ docker-run-circle:
 		-e ADVERTISED_PORT=9092 \
 		-p "2181:2181" -p "9092:9092" \
 		-d spotify/kafka
+	docker run --name elasticsearch -p "9200:9200" -p "9300:9300" -d elasticsearch:5
 	docker run --name nsq -p "4150:4150" -d nsqio/nsq /nsqd
 	docker run --name mqtt -p "1883:1883" -d ncarlier/mqtt
 	docker run --name riemann -p "5555:5555" -d blalor/riemann
 	docker run --name nats -p "4222:4222" -d nats
 
 # Kill all docker containers, ignore errors
 docker-kill:
-	-docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats
-	-docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats
+	-docker kill nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats elasticsearch
+	-docker rm nsq aerospike redis rabbitmq postgres memcached mysql kafka mqtt riemann nats elasticsearch
 
 # Run full unit tests using docker containers (includes setup and teardown)
 test: vet docker-kill docker-run

README.md

@@ -207,6 +207,7 @@ Telegraf can also collect metrics via the following service plugins:
 * [aws cloudwatch](./plugins/outputs/cloudwatch)
 * [datadog](./plugins/outputs/datadog)
 * [discard](./plugins/outputs/discard)
+* [elasticsearch](./plugins/outputs/elasticsearch)
 * [file](./plugins/outputs/file)
 * [graphite](./plugins/outputs/graphite)
 * [graylog](./plugins/outputs/graylog)

plugins/outputs/all/all.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch"
 	_ "github.com/influxdata/telegraf/plugins/outputs/datadog"
 	_ "github.com/influxdata/telegraf/plugins/outputs/discard"
+	_ "github.com/influxdata/telegraf/plugins/outputs/elasticsearch"
 	_ "github.com/influxdata/telegraf/plugins/outputs/file"
 	_ "github.com/influxdata/telegraf/plugins/outputs/graphite"
 	_ "github.com/influxdata/telegraf/plugins/outputs/graylog"

plugins/outputs/elasticsearch/README.md

@@ -0,0 +1,131 @@
+## Elasticsearch Output Plugin for Telegraf
+
+This plugin writes to [Elasticsearch](https://www.elastic.co) via HTTP using Elastic (http://olivere.github.io/elastic/).
+
+It only supports Elasticsearch 5.x series currently.
+
+## Elasticsearch indexes and templates
+
+### Indexes per time-frame
+
+This plugin can manage indexes per time-frame, as commonly done in other tools with Elasticsearch.
+
+The timestamp of the metric collected will be used to decide the index destination.
+
+For more information about this usage on Elasticsearch, check https://www.elastic.co/guide/en/elasticsearch/guide/master/time-based.html#index-per-timeframe
+
+### Template management
+
+Index templates are used in Elasticsearch to define settings and mappings for the indexes and how the fields should be analyzed.
+For more information on how this works, see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
+
+This plugin can create a recommended template for use with telegraf metrics. If it already exists, it will not overwrite unless you configure this plugin to do so.
+
+### Example events:
+
+This plugin will format the events in the following way:
+
+```json
+{
+  "@timestamp": "2017-01-01T00:00:00+00:00",
+  "input_plugin": "cpu",
+  "cpu": {
+    "usage_guest": 0,
+    "usage_guest_nice": 0,
+    "usage_idle": 71.85413456197966,
+    "usage_iowait": 0.256805341656516,
+    "usage_irq": 0,
+    "usage_nice": 0,
+    "usage_softirq": 0.2054442732579466,
+    "usage_steal": 0,
+    "usage_system": 15.04879301548127,
+    "usage_user": 12.634822807288275
+  },
+  "tag": {
+    "cpu": "cpu-total",
+    "host": "elastichost",
+    "dc": "datacenter1"
+  }
+}
+```
+
+```json
+{
+  "@timestamp": "2017-01-01T00:00:00+00:00",
+  "input_plugin": "system",
+  "system": {
+    "load1": 0.78,
+    "load15": 0.8,
+    "load5": 0.8,
+    "n_cpus": 2,
+    "n_users": 2
+  },
+  "tag": {
+    "host": "elastichost",
+    "dc": "datacenter1"
+  }
+}
+```
+
+### Configuration:
+
+```toml
+# Configuration for Elasticsearch to send metrics to.
+[[outputs.elasticsearch]]
+  ## The full HTTP endpoint URL for your Elasticsearch instance
+  ## Multiple urls can be specified as part of the same cluster,
+  ## this means that only ONE of the urls will be written to each interval.
+  urls = [ "http://node1.es.example.com:9200" ] # required.
+  ## Set to true to ask Elasticsearch a list of all cluster nodes,
+  ## thus it is not necessary to list all nodes in the urls config option
+  enable_sniffer = true
+  ## Set the interval to check if the nodes are available, in seconds.
+  ## Setting to 0 will disable the health check (not recommended in production)
+  health_check_interval = 10
+  ## HTTP basic authentication details (eg. when using Shield)
+  # username = "telegraf"
+  # password = "mypassword"
+
+  # Index Config
+  ## The target index for metrics (Elasticsearch will create if it not exists).
+  ## You can use the date specifiers below to create indexes per time frame.
+  ## The metric timestamp will be used to decide the destination index name
+  # %Y - year (2016)
+  # %y - last two digits of year (00..99)
+  # %m - month (01..12)
+  # %d - day of month (e.g., 01)
+  # %H - hour (00..23)
+  index_name = "telegraf-%Y.%m.%d" # required.
+
+  ## Template Config
+  ## Set to true if you want telegraf to manage its index template.
+  ## If enabled it will create a recommended index template for telegraf indexes
+  manage_template = true
+  ## The template name used for telegraf indexes
+  template_name = "telegraf"
+  ## Set to true if you want to overwrite an existing template
+  overwrite_template = false
+```
+
+### Required parameters:
+
+* `urls`: A list containing the full HTTP URL of one or more nodes from your Elasticsearch instance.
+* `index_name`: The target index for metrics. You can use the date specifiers below to create indexes per time frame.
+
+``` 
+  %Y - year (2017)
+  %y - last two digits of year (00..99)
+  %m - month (01..12)
+  %d - day of month (e.g., 01)
+  %H - hour (00..23)
+```
+
+### Optional parameters:
+
+* `enable_sniffer`: Set to true to ask Elasticsearch a list of all cluster nodes, thus it is not necessary to list all nodes in the urls config option.
+* `health_check_interval`: Set the interval to check if the nodes are available, in seconds. Setting to 0 will disable the health check (not recommended in production).
+* `username`: The username for HTTP basic authentication details (eg. when using Shield).
+* `password`: The password for HTTP basic authentication details (eg. when using Shield).
+* `manage_template`: Set to true if you want telegraf to manage its index template. If enabled it will create a recommended index template for telegraf indexes.
+* `template_name`: The template name used for telegraf indexes.
+* `overwrite_template`: Set to true if you want to overwrite an existing template.