Create jvm fileset (elastic#7305)

Adding fileset JVM GC logs for Elasticsearch Filebeat module. This baseline fileset does not do much yet, more to come. 

elastic#5301.

CHANGELOG.asciidoc

@@ -192,6 +192,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Correctly join partial log lines when using `docker` input. {pull}6967[6967]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
 - Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
+- Add GC fileset to the Elasticsearch module. {pull}7305[7305]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -800,6 +800,42 @@ elasticsearch Module
 
 
 
+[float]
+== gc fields
+
+
+
+
+*`elasticsearch.gc.relative_process_timestamp_secs`*::
++
+--
+type: float
+
+Garbage collection threads total stop time seconds.
+
+
+--
+
+*`elasticsearch.gc.threads_total_stop_time_secs`*::
++
+--
+type: float
+
+Garbage collection threads total stop time seconds.
+
+
+--
+
+*`elasticsearch.gc.stopping_threads_time_secs`*::
++
+--
+type: float
+
+Time took to stop threads seconds.
+
+
+--
+
 [float]
 == server fields
 

filebeat/filebeat.reference.yml

@@ -92,6 +92,12 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  gc:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Icinga Module -------------------------------
 #- module: icinga
   # Main logs

filebeat/module/elasticsearch/_meta/config.yml

@@ -6,3 +6,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  gc:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

filebeat/module/elasticsearch/gc/_meta/fields.yml

@@ -0,0 +1,16 @@
+- name: gc
+  type: group
+  description: >
+  fields:
+    - name: relative_process_timestamp_secs
+      type: float
+      description: >
+        Garbage collection threads total stop time seconds.
+    - name: threads_total_stop_time_secs
+      type: float
+      description: >
+        Garbage collection threads total stop time seconds.
+    - name: stopping_threads_time_secs
+      type: float
+      description: >
+        Time took to stop threads seconds.

filebeat/module/elasticsearch/gc/config/gc.yml

@@ -0,0 +1,14 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+multiline:
+  pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  negate: true
+  match: after
+
+fields:
+  service.name: "elasticsearch"
+fields_under_root: true

filebeat/module/elasticsearch/gc/ingest/pipeline.json

@@ -0,0 +1,39 @@
+{
+  "description": "Pipeline for parsing Elasticsearch JVM garbage collection logs",
+  "processors": [
+    {
+      "grok": {
+        "field": "message",
+        "patterns": [
+          "%{GCTIMESTAMP}: %{GCPROCRUNTIME}: Total time for which application threads were stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_secs} seconds, Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_secs} seconds",
+          "%{GCTIMESTAMP}: %{GREEDYMULTILINE:message}"
+        ],
+        "pattern_definitions": {
+          "GREEDYMULTILINE": "(.|\n)*",
+          "GCTIMESTAMP": "%{TIMESTAMP_ISO8601:timestamp}",
+          "GCPROCRUNTIME": "%{BASE10NUM:elasticsearch.gc.relative_process_timestamp_secs}"
+        }
+      }
+    },
+    {
+      "rename": {
+        "field": "@timestamp",
+        "target_field": "event.created"
+      }
+    },
+    {
+      "rename": {
+        "field": "timestamp",
+        "target_field": "@timestamp"
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "set": {
+        "field": "error.message",
+        "value": "{{ _ingest.on_failure_message }}"
+      }
+    }
+  ]
+}

filebeat/module/elasticsearch/gc/manifest.yml

@@ -0,0 +1,11 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/elasticsearch/gc.log.[0-9]*
+    os.darwin: []
+    os.windows: []
+
+ingest_pipeline: ingest/pipeline.json
+input: config/gc.yml

filebeat/module/elasticsearch/gc/test/test.log

@@ -0,0 +1,3 @@
+2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds
+
+

filebeat/module/elasticsearch/gc/test/test.log-expected.json

@@ -0,0 +1,45 @@
+[
+  {
+    "_index": "filebeat-7.0.0-alpha1-2018.06.15",
+    "_type": "doc",
+    "_id": "w-p9BWQBiF0NvuCjhcyk",
+    "_score": 1,
+    "_source": {
+      "offset": 160,
+      "prospector": {
+        "type": "log"
+      },
+      "source": "/Users/jbryan/dev/go/src/github.com/elastic/beats/filebeat/module/elasticsearch/gc/test/test.log",
+      "message": "2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds\n\n",
+      "fileset": {
+        "module": "elasticsearch",
+        "name": "gc"
+      },
+      "input": {
+        "type": "log"
+      },
+      "@timestamp": "2018-06-11T01:53:11.382+0000",
+      "elasticsearch": {
+        "gc": {
+          "stopping_threads_time_secs": "0.0000702",
+          "relative_process_timestamp_secs": "1396138.752",
+          "threads_total_stop_time_secs": "0.0083760"
+        }
+      },
+      "service": {
+        "name": "elasticsearch"
+      },
+      "host": {
+        "name": "peanut.local"
+      },
+      "beat": {
+        "hostname": "peanut.local",
+        "name": "peanut.local",
+        "version": "7.0.0-alpha1"
+      },
+      "event": {
+        "created": "2018-06-15T22:07:43.170Z"
+      }
+    }
+  }
+]

filebeat/module/elasticsearch/module.yml

@@ -1,2 +1,3 @@
 dashboards:
 
+

filebeat/modules.d/elasticsearch.yml.disabled

@@ -6,3 +6,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  gc:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths: