From 65fafea18f7c99db9d314b11715e27c0154ff396 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 16 Apr 2020 09:33:42 -0700 Subject: [PATCH] Handle ECS-compatible server logs emitted by ES 8.0.0+ (#17714) * Handle ECS-compatible server logs emitted by ES 8.0.0+ * Adding CHANGELOG entry * Adding a couple more log entries --- CHANGELOG.next.asciidoc | 1 + .../server/ingest/pipeline-json.yml | 44 +++++++++++- .../elasticsearch/server/ingest/pipeline.yml | 5 +- .../server/test/elasticsearch-json.800.log | 3 + .../elasticsearch-json.800.log-expected.json | 71 +++++++++++++++++++ 5 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log create mode 100644 filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c0d9010422d..3cb901afe82 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -255,6 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] +- Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714] *Heartbeat* diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml index 1f2022f0b65..c3b655643ed 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -11,12 +11,17 @@ processors: if: ctx.elasticsearch.server.type != 'server' - remove: field: elasticsearch.server.type +- dot_expander: + field: service.name + path: elasticsearch.server - rename: - field: elasticsearch.server.level - target_field: log.level + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true - rename: field: elasticsearch.server.component target_field: elasticsearch.component + ignore_missing: true - dot_expander: field: cluster.name path: elasticsearch.server @@ -43,6 +48,31 @@ processors: field: elasticsearch.server.node.id target_field: elasticsearch.node.id ignore_missing: true +- rename: + field: elasticsearch.server.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true - grok: field: elasticsearch.server.message pattern_definitions: @@ -60,9 +90,17 @@ processors: - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} - remove: field: elasticsearch.server.message -- date: +- rename: + field: elasticsearch.server.@timestamp + target_field: '@timestamp' + ignore_missing: true +- rename: field: elasticsearch.server.timestamp target_field: '@timestamp' + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' formats: - ISO8601 ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 032e3581d0b..786a8484d42 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -80,7 +80,10 @@ processors: - elasticsearch.server.gc.observation_duration.unit ignore_missing: true - remove: - field: elasticsearch.server.timestamp + field: + - elasticsearch.server.timestamp + - elasticsearch.server.@timestamp + ignore_missing: true - remove: field: - first_char diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log new file mode 100644 index 00000000000..b7119ffc069 --- /dev/null +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log @@ -0,0 +1,3 @@ +{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"} +{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} +{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json new file mode 100644 index 00000000000..817cadf6002 --- /dev/null +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json @@ -0,0 +1,71 @@ +[ + { + "@timestamp": "2020-04-14T14:05:58.019Z", + "elasticsearch.cluster.name": "elasticsearch", + "elasticsearch.cluster.uuid": "ECEBF2VPQuCF9tbBKaLqXQ", + "elasticsearch.node.id": "suOYiQwuRvialOY-c0wHLA", + "elasticsearch.node.name": "CBR-MBP.local", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "suOYiQwuRvialOY-c0wHLA", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService", + "log.offset": 0, + "message": "adding template [.management-beats] for index patterns [.management-beats]", + "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-14T20:57:49.663Z", + "elasticsearch.cluster.name": "docker-cluster", + "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", + "elasticsearch.index.name": "test-filebeat-modules", + "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", + "elasticsearch.node.name": "7debcb878699", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "kZnYdakGTqihZQT_1rM92g", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataCreateIndexService", + "log.offset": 489, + "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", + "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-14T20:57:49.772Z", + "elasticsearch.cluster.name": "docker-cluster", + "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", + "elasticsearch.index.id": "IW1jJcOBTFeIDihqjoT8yQ", + "elasticsearch.index.name": "test-filebeat-modules", + "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", + "elasticsearch.node.name": "7debcb878699", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "kZnYdakGTqihZQT_1rM92g", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataMappingService", + "log.offset": 1031, + "message": "update_mapping [_doc]", + "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + } +] \ No newline at end of file