[logstash] align mappings with metricbeat (#3993)

* ecs fields

* node data stream fields

* node_stats data stream fields

* _dev/deploy configuration

* Update packages/logstash/_dev/deploy/docker/config/logstash.yml

Co-authored-by: Mat Schaffer <mat@schaffer.me>

Co-authored-by: Mat Schaffer <mat@schaffer.me>

packages/logstash/_dev/deploy/docker/config/logstash.yml

@@ -0,0 +1,2 @@
+http.host: "0.0.0.0"
+config.reload.automatic: true

packages/logstash/_dev/deploy/docker/config/pipelines.yml

@@ -0,0 +1,7 @@
+- pipeline.id: pipeline-with-persisted-queue
+  path.config: "/usr/share/logstash/pipeline/persisted-queue.conf"
+  queue.type: persisted
+- pipeline.id: pipeline-with-memory-queue
+  path.config: "/usr/share/logstash/pipeline/memory-queue.conf"
+- pipeline.id: standalone-pipeline
+  path.config: "/usr/share/logstash/pipeline/standalone-pipeline.conf"

packages/logstash/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '2.3'
+services:
+  logstash:
+    image: "docker.elastic.co/logstash/logstash:8.5.0-SNAPSHOT"
+    volumes:
+      - "./pipeline:/usr/share/logstash/pipeline"
+      - "./config:/usr/share/logstash/config"
+    ports:
+      - "127.0.0.1:9600:9600"

packages/logstash/_dev/deploy/docker/pipeline/memory-queue.conf

@@ -0,0 +1,15 @@
+input {
+  java_generator {
+    eps => 10
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    index => "logstash-memory-queue"
+    ssl_certificate_verification => "false"
+  }
+}

packages/logstash/_dev/deploy/docker/pipeline/persisted-queue.conf

@@ -0,0 +1,15 @@
+input {
+  java_generator {
+    eps => 50
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => ["https://elasticsearch:9200"]
+    user => "elastic"
+    password => "changeme"
+    index => "logstash-persisted-queue"
+    ssl_certificate_verification => "false"
+  }
+}

packages/logstash/_dev/deploy/docker/pipeline/standalone-pipeline.conf

@@ -0,0 +1,9 @@
+input {
+  java_generator {
+    eps => 5
+  }
+}
+
+output {
+  stdout { }
+}

packages/logstash/data_stream/node/fields/ecs.yml

@@ -15,3 +15,11 @@
   external: ecs
 - name: ecs.version
   external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: host.name
+  external: ecs

packages/logstash/data_stream/node/fields/fields.yml

@@ -1,45 +1,49 @@
-- name: logstash.node
+- name: logstash
   type: group
-  release: ga
   fields:
-    - name: host
+    - name: elasticsearch.cluster.id
       type: keyword
-    - name: version
-      type: keyword
-    - name: id
-      type: keyword
-    - name: state.pipeline
+    - name: node
       type: group
       fields:
-        - name: id
+        - name: jvm
+          type: group
+          fields:
+            - name: version
+              type: keyword
+              ignore_above: 1024
+        - name: host
           type: keyword
-        - name: hash
+        - name: version
           type: keyword
-        - name: ephemeral_id
+        - name: id
           type: keyword
-        - name: batch_size
-          type: long
-        - name: workers
-          type: long
-        - name: representation
+        - name: state.pipeline
           type: group
           fields:
-            - name: hash
+            - name: id
               type: keyword
-            - name: type
+            - name: hash
               type: keyword
-            - name: version
+            - name: ephemeral_id
               type: keyword
-            - name: graph
+            - name: batch_size
+              type: long
+            - name: workers
+              type: long
+            - name: representation
               type: group
               fields:
-                - name: edges
-                  type: object
-                - name: vertices
-                  type: object
-    - name: jvm
-      type: group
-      fields:
-        - name: version
-          type: keyword
-          description: JVM version
+                - name: hash
+                  type: keyword
+                - name: type
+                  type: keyword
+                - name: version
+                  type: keyword
+                - name: graph
+                  type: group
+                  fields:
+                    - name: edges
+                      type: object
+                    - name: vertices
+                      type: object

packages/logstash/data_stream/node/fields/package-fields.yml

@@ -1,6 +1,15 @@
+- name: cluster_uuid
+  type: alias
+  path: logstash.elasticsearch.cluster.id
+- name: timestamp
+  type: alias
+  path: '@timestamp'
 - name: logstash_stats
   type: group
   fields:
+    - name: timestamp
+      type: alias
+      path: '@timestamp'
     - name: jvm
       type: group
       fields:
@@ -37,8 +46,6 @@
         - name: version
           type: alias
           path: logstash.node.stats.logstash.version
-    - name: pipelines
-      type: nested
     - name: os
       type: group
       fields:
@@ -90,5 +97,3 @@
     - name: pipeline.hash
       type: alias
       path: logstash.node.state.pipeline.hash
-- name: logstash
-  type: group

packages/logstash/data_stream/node/manifest.yml

@@ -1,6 +1,10 @@
 type: metrics
 title: Logstash node metrics
 release: experimental
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
 streams:
   - input: logstash/metrics
     title: Logstash node metrics

packages/logstash/data_stream/node_stats/fields/ecs.yml

@@ -1,8 +1,8 @@
 - name: '@timestamp'
   external: ecs
-- name: host.hostname
+- name: service.id
   external: ecs
-- name: process.pid
+- name: service.type
   external: ecs
 - name: service.version
   external: ecs
@@ -11,9 +11,15 @@
   description: Address where data about this service was collected from.
 - name: service.name
   external: ecs
-- name: service.id
-  external: ecs
-- name: service.type
+- name: process.pid
   external: ecs
 - name: ecs.version
   external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: host.name
+  external: ecs