Handle ECS-compatible server logs emitted by ES 8.0.0+

CHANGELOG.next.asciidoc

@@ -255,6 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491]
 - Release Google Cloud module as GA. {pull}17511[17511]
 - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550]
+- Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
 
 *Heartbeat*
 

filebeat/module/elasticsearch/server/ingest/pipeline-json.yml

@@ -11,12 +11,17 @@ processors:
     if: ctx.elasticsearch.server.type != 'server'
 - remove:
     field: elasticsearch.server.type
+- dot_expander:
+    field: service.name
+    path: elasticsearch.server
 - rename:
-    field: elasticsearch.server.level
-    target_field: log.level
+    field: elasticsearch.server.service.name
+    target_field: service.name
+    ignore_missing: true
 - rename:
     field: elasticsearch.server.component
     target_field: elasticsearch.component
+    ignore_missing: true
 - dot_expander:
     field: cluster.name
     path: elasticsearch.server
@@ -43,6 +48,31 @@ processors:
     field: elasticsearch.server.node.id
     target_field: elasticsearch.node.id
     ignore_missing: true
+- rename:
+    field: elasticsearch.server.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.level
+    path: elasticsearch.server
+- rename:
+    field: elasticsearch.server.log.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.logger
+    path: elasticsearch.server
+- rename:
+    field: elasticsearch.server.log.logger
+    target_field: log.logger
+    ignore_missing: true
+- dot_expander:
+    field: process.thread.name
+    path: elasticsearch.server
+- rename:
+    field: elasticsearch.server.process.thread.name
+    target_field: process.thread.name
+    ignore_missing: true
 - grok:
     field: elasticsearch.server.message
     pattern_definitions:
@@ -60,9 +90,17 @@ processors:
     - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message}
 - remove:
     field: elasticsearch.server.message
-- date:
+- rename:
+    field: elasticsearch.server.@timestamp
+    target_field: '@timestamp'
+    ignore_missing: true
+- rename:
     field: elasticsearch.server.timestamp
     target_field: '@timestamp'
+    ignore_missing: true
+- date:
+    field: '@timestamp'
+    target_field: '@timestamp'
     formats:
     - ISO8601
     ignore_failure: true

filebeat/module/elasticsearch/server/ingest/pipeline.yml

@@ -80,7 +80,10 @@ processors:
     - elasticsearch.server.gc.observation_duration.unit
     ignore_missing: true
 - remove:
-    field: elasticsearch.server.timestamp
+    field:
+    - elasticsearch.server.timestamp
+    - elasticsearch.server.@timestamp
+    ignore_missing: true
 - remove:
     field:
     - first_char

filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log

@@ -0,0 +1,3 @@
+{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"}
+{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"}
+{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"}

filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json

@@ -0,0 +1,71 @@
+[
+    {
+        "@timestamp": "2020-04-14T14:05:58.019Z",
+        "elasticsearch.cluster.name": "elasticsearch",
+        "elasticsearch.cluster.uuid": "ECEBF2VPQuCF9tbBKaLqXQ",
+        "elasticsearch.node.id": "suOYiQwuRvialOY-c0wHLA",
+        "elasticsearch.node.name": "CBR-MBP.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.server",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "server",
+        "host.id": "suOYiQwuRvialOY-c0wHLA",
+        "input.type": "log",
+        "log.level": "INFO",
+        "log.logger": "org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService",
+        "log.offset": 0,
+        "message": "adding template [.management-beats] for index patterns [.management-beats]",
+        "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2020-04-14T20:57:49.663Z",
+        "elasticsearch.cluster.name": "docker-cluster",
+        "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ",
+        "elasticsearch.index.name": "test-filebeat-modules",
+        "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g",
+        "elasticsearch.node.name": "7debcb878699",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.server",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "server",
+        "host.id": "kZnYdakGTqihZQT_1rM92g",
+        "input.type": "log",
+        "log.level": "INFO",
+        "log.logger": "org.elasticsearch.cluster.metadata.MetadataCreateIndexService",
+        "log.offset": 489,
+        "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]",
+        "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2020-04-14T20:57:49.772Z",
+        "elasticsearch.cluster.name": "docker-cluster",
+        "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ",
+        "elasticsearch.index.id": "IW1jJcOBTFeIDihqjoT8yQ",
+        "elasticsearch.index.name": "test-filebeat-modules",
+        "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g",
+        "elasticsearch.node.name": "7debcb878699",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.server",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "server",
+        "host.id": "kZnYdakGTqihZQT_1rM92g",
+        "input.type": "log",
+        "log.level": "INFO",
+        "log.logger": "org.elasticsearch.cluster.metadata.MetadataMappingService",
+        "log.offset": 1031,
+        "message": "update_mapping [_doc]",
+        "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]",
+        "service.name": "ES_ECS",
+        "service.type": "elasticsearch"
+    }
+]