From 89d96511ab89233441a445c752aa5954de935d5a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 14 Apr 2020 13:58:23 -0700 Subject: [PATCH 1/3] Handle ECS-compatible server logs emitted by ES 8.0.0+ --- .../server/ingest/pipeline-json.yml | 44 +++++++++++++++++-- .../elasticsearch/server/ingest/pipeline.yml | 5 ++- .../server/test/elasticsearch-json.800.log | 1 + .../elasticsearch-json.800.log-expected.json | 24 ++++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log create mode 100644 filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml index 1f2022f0b65..c3b655643ed 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -11,12 +11,17 @@ processors: if: ctx.elasticsearch.server.type != 'server' - remove: field: elasticsearch.server.type +- dot_expander: + field: service.name + path: elasticsearch.server - rename: - field: elasticsearch.server.level - target_field: log.level + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true - rename: field: elasticsearch.server.component target_field: elasticsearch.component + ignore_missing: true - dot_expander: field: cluster.name path: elasticsearch.server @@ -43,6 +48,31 @@ processors: field: elasticsearch.server.node.id target_field: elasticsearch.node.id ignore_missing: true +- rename: + field: elasticsearch.server.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true - grok: field: elasticsearch.server.message pattern_definitions: @@ -60,9 +90,17 @@ processors: - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} - remove: field: elasticsearch.server.message -- date: +- rename: + field: elasticsearch.server.@timestamp + target_field: '@timestamp' + ignore_missing: true +- rename: field: elasticsearch.server.timestamp target_field: '@timestamp' + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' formats: - ISO8601 ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 032e3581d0b..786a8484d42 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -80,7 +80,10 @@ processors: - elasticsearch.server.gc.observation_duration.unit ignore_missing: true - remove: - field: elasticsearch.server.timestamp + field: + - elasticsearch.server.timestamp + - elasticsearch.server.@timestamp + ignore_missing: true - remove: field: - first_char diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log new file mode 100644 index 00000000000..3cf7fc74454 --- /dev/null +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log @@ -0,0 +1 @@ +{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"} diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json new file mode 100644 index 00000000000..9cc215e50b8 --- /dev/null +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json @@ -0,0 +1,24 @@ +[ + { + "@timestamp": "2020-04-14T14:05:58.019Z", + "elasticsearch.cluster.name": "elasticsearch", + "elasticsearch.cluster.uuid": "ECEBF2VPQuCF9tbBKaLqXQ", + "elasticsearch.node.id": "suOYiQwuRvialOY-c0wHLA", + "elasticsearch.node.name": "CBR-MBP.local", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "suOYiQwuRvialOY-c0wHLA", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService", + "log.offset": 0, + "message": "adding template [.management-beats] for index patterns [.management-beats]", + "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + } +] \ No newline at end of file From c666855d40e99ae488dc3dc28a47907efd0f67b5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 14 Apr 2020 14:10:31 -0700 Subject: [PATCH 2/3] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c0d9010422d..3cb901afe82 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -255,6 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] +- Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714] *Heartbeat* From a9bc7b762c0fc2e59bf9118da073f88e0d806755 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 14 Apr 2020 14:15:42 -0700 Subject: [PATCH 3/3] Adding a couple more log entries --- .../server/test/elasticsearch-json.800.log | 2 + .../elasticsearch-json.800.log-expected.json | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log index 3cf7fc74454..b7119ffc069 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log @@ -1 +1,3 @@ {"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"} +{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} +{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json index 9cc215e50b8..817cadf6002 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json @@ -20,5 +20,52 @@ "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-14T20:57:49.663Z", + "elasticsearch.cluster.name": "docker-cluster", + "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", + "elasticsearch.index.name": "test-filebeat-modules", + "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", + "elasticsearch.node.name": "7debcb878699", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "kZnYdakGTqihZQT_1rM92g", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataCreateIndexService", + "log.offset": 489, + "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", + "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" + }, + { + "@timestamp": "2020-04-14T20:57:49.772Z", + "elasticsearch.cluster.name": "docker-cluster", + "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", + "elasticsearch.index.id": "IW1jJcOBTFeIDihqjoT8yQ", + "elasticsearch.index.name": "test-filebeat-modules", + "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", + "elasticsearch.node.name": "7debcb878699", + "event.category": "database", + "event.dataset": "elasticsearch.server", + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "server", + "host.id": "kZnYdakGTqihZQT_1rM92g", + "input.type": "log", + "log.level": "INFO", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataMappingService", + "log.offset": 1031, + "message": "update_mapping [_doc]", + "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "service.name": "ES_ECS", + "service.type": "elasticsearch" } ] \ No newline at end of file