diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml new file mode 100644 index 0000000000000..38d19cab13d30 --- /dev/null +++ b/docs/changelog/109025.yaml @@ -0,0 +1,6 @@ +pr: 109025 +summary: Introduce a setting controlling the activation of the `logs` index mode in logs@settings +area: Logs +type: feature +issues: + - 108762 diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 8a343ff9cf853..f95d9a0b0431f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -62,7 +62,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -86,7 +86,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -110,7 +110,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String LOG_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", @@ -121,7 +121,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String TIME_SERIES_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "ip_address": "%s", @@ -207,7 +207,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -224,7 +224,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut assertDataStreamBackingIndicesModes(dataStreamName, List.of(IndexMode.LOGS, IndexMode.TIME_SERIES, IndexMode.LOGS)); } - public void testInvalidIndexModeTimeSeriesSwitchWithoutROutingPath() throws IOException, ExecutionException, InterruptedException { + public void testInvalidIndexModeTimeSeriesSwitchWithoutRoutingPath() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); @@ -250,7 +250,7 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -269,8 +269,9 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc assertThat( exception.getCause().getCause().getMessage(), Matchers.equalTo( - "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with " - + "a list of dimensions in [time_series_dimensions] and without the [script] parameter. [hostname] was not a dimension." + "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields " + + "with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a " + + "dimension." ) ); } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java index c18bcf750242f..d3ec5b29ff5b9 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java @@ -80,7 +80,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -116,7 +116,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -138,7 +138,7 @@ private static void waitForLogs(RestClient client) throws Exception { private static final String DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java new file mode 100644 index 0000000000000..dcd2457b88f18 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + public void testLogsSettingsIndexModeDisabled() throws IOException { + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java new file mode 100644 index 0000000000000..832267cebf97c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + private static final String ALTERNATE_HOST_MAPPING = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host.cloud_region": { + "type": "keyword" + }, + "host.availability_zone": { + "type": "keyword" + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "subobjects": false, + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + + private static String BULK_INDEX_REQUEST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + """; + + private static String BULK_INDEX_REQUEST_WITH_HOST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } + """; + + public void testCreateDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + } + + public void testBulkIndexing() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithFlatHostProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", ALTERNATE_HOST_MAPPING)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostDefaultSubobjectsProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostSubobjectsFalseProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testRolloverDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); + assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); + assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java new file mode 100644 index 0000000000000..ff45096146280 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { + protected static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + final Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + throws IOException { + final Request request = new Request("PUT", "/_component_template/" + templateName); + request.setJsonEntity(mappings); + return client.performRequest(request); + } + + protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); + } + + protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); + } + + @SuppressWarnings("unchecked") + protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) + throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.get(backingIndex).get("index_name"); + } + + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); + final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) + .get("settings"); + + return settings.get(setting); + } + + protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) + throws IOException { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + bulkRequest.setJsonEntity(bulkSupplier.get()); + bulkRequest.addParameter("refresh", "true"); + return client.performRequest(bulkRequest); + } +} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 128903f4faac8..5e8948b7fdea3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -31,7 +31,7 @@ create logs index: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -48,17 +48,17 @@ create logs index: refresh: true body: - { "index": { } } - - { "@timestamp": "2024-02-12T10:30:00Z", "hostname": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } + - { "@timestamp": "2024-02-12T10:30:00Z", ignored_field_stats: "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:31:00Z", "hostname": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:32:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:33:00Z", "hostname": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:34:00Z", "hostname": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:35:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } - do: @@ -103,7 +103,7 @@ using default timestamp field mapping: number_of_shards: 2 mappings: properties: - hostname: + host.name: type: keyword agent_id: type: keyword @@ -149,7 +149,7 @@ missing hostname field: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[hostname]" } + - match: { error.reason: "unknown index sort field:[host.name]" } --- missing sort field: @@ -177,7 +177,7 @@ missing sort field: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -271,7 +271,7 @@ override sort order settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -319,7 +319,7 @@ override sort missing settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -367,7 +367,7 @@ override sort mode settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -410,12 +410,12 @@ override sort field using nested field type in sorting: number_of_replicas: 0 number_of_shards: 2 sort: - field: [ "hostname", "nested", "@timestamp" ] + field: [ "host.name", "nested", "@timestamp" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -459,7 +459,7 @@ override sort field using nested field type: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -499,12 +499,12 @@ routing path not allowed in logs mode: mode: logs number_of_replicas: 0 number_of_shards: 2 - routing_path: [ "hostname", "agent_id" ] + routing_path: [ "host.name", "agent_id" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -545,7 +545,7 @@ start time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -586,7 +586,7 @@ end time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 74c2c57594e72..f190462d6d1e9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -154,7 +154,7 @@ public IndexSortConfig(IndexSettings indexSettings) { List fields = INDEX_SORT_FIELD_SETTING.get(settings); if (this.indexMode == IndexMode.LOGS && fields.isEmpty()) { - fields = List.of("hostname", DataStream.TIMESTAMP_FIELD_NAME); + fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME); } this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json new file mode 100644 index 0000000000000..167efbd3ffaf5 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json @@ -0,0 +1,31 @@ +{ + "template": { + "mappings": { + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + }, + "_meta": { + "description": "default mappings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json new file mode 100644 index 0000000000000..b02866e867c4a --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "mode": "logs", + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index 2577cf28f4213..cc127883652af 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -12,7 +12,6 @@ import org.elasticsearch.plugins.Plugin; import java.util.Collection; -import java.util.Collections; import java.util.List; public class StackPlugin extends Plugin implements ActionPlugin { @@ -24,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return Collections.singletonList(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 3cd551ca1f3d9..34cacbb8956e5 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -57,10 +57,21 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); + /** + * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' + */ + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( + "cluster.logsdb.enabled", + false, + Setting.Property.NodeScope + ); + private final ClusterService clusterService; private final FeatureService featureService; private volatile boolean stackTemplateEnabled; + private final boolean logsIndexModeTemplateEnabled; + public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream @@ -121,6 +132,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); + this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED.get(nodeSettings); } @Override @@ -164,6 +176,7 @@ protected List getLifecyclePolicies() { } private static final Map COMPONENT_TEMPLATE_CONFIGS; + private static final Map LOGSDB_COMPONENT_TEMPLATE_CONFIGS; static { final Map componentTemplates = new HashMap<>(); @@ -249,10 +262,97 @@ protected List getLifecyclePolicies() { } } COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates); + + final Map logsdbComponentTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/data-streams@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/logs@mappings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + ECS_DYNAMIC_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/ecs@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs@settings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_TSDB_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@tsdb-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, + "/kibana-reporting@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ) + )) { + try { + logsdbComponentTemplates.put( + config.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) + ); + } catch (IOException e) { + throw new AssertionError(e); + } + } + LOGSDB_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(logsdbComponentTemplates); } @Override protected Map getComponentTemplateConfigs() { + if (logsIndexModeTemplateEnabled) { + return LOGSDB_COMPONENT_TEMPLATE_CONFIGS; + } return COMPONENT_TEMPLATE_CONFIGS; }