From ac4bd1a3aaf67fa89d3a5591a835cc3dce259c8a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 May 2024 10:54:08 +0200 Subject: [PATCH 01/54] feature: enable logs index mode for logs behind feature flag --- .../test/cluster/FeatureFlag.java | 3 ++- .../src/main/resources/logs@settings-logsdb | 26 +++++++++++++++++++ .../xpack/stack/StackTemplateRegistry.java | 14 ++++++++-- 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index d555337f467ae..7ff978fcbf6ef 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -17,7 +17,8 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - SEMANTIC_TEXT_ENABLED("es.semantic_text_feature_flag_enabled=true", Version.fromString("8.15.0"), null); + SEMANTIC_TEXT_ENABLED("es.semantic_text_feature_flag_enabled=true", Version.fromString("8.15.0"), null), + LOGS_INDEX_MODE("es.logs_index_mode_feature_flag_registered=true", Version.fromString("8.15.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb new file mode 100644 index 0000000000000..b02866e867c4a --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "mode": "logs", + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 30323a1d7d363..28e435b852af1 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -167,7 +167,7 @@ protected List getLifecyclePolicies() { static { final Map componentTemplates = new HashMap<>(); - for (IndexTemplateConfig config : List.of( + List configs = List.of( new IndexTemplateConfig( DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, "/data-streams@mappings.json", @@ -238,7 +238,17 @@ protected List getLifecyclePolicies() { TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES ) - )) { + ); + configs.add( + new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs@settings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ) + ); + for (IndexTemplateConfig config : configs) { try { componentTemplates.put( config.getTemplateName(), From bcf8d75506bfc93ca4e9a487d4ae1c9c5a19d731 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 May 2024 16:10:18 +0200 Subject: [PATCH 02/54] feature: introduce a feature flag and a setting controlling 'logs' index mode --- server/build.gradle | 2 + .../org/elasticsearch/index/IndexMode.java | 71 +++++++++++++++++++ .../elasticsearch/index/IndexSettings.java | 26 ++++++- .../xpack/stack/StackTemplateRegistry.java | 45 ++++++++---- 4 files changed, 131 insertions(+), 13 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 03713bc3d2837..74672ba2f144a 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -145,10 +145,12 @@ if (BuildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' + systemProperty 'es.index_mode_logs_feature_flag_enabled', 'true' } tasks.named("internalClusterTest").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' + systemProperty 'es.index_mode_logs_feature_flag_enabled', 'true' } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 05169836d6617..4605e0c5257ad 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -229,6 +229,77 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { public boolean isSyntheticSourceEnabled() { return true; } + }, + LOGS("logs") { // TODO: stub waiting for PR #108896 + @Override + void validateWithOtherSettings(Map, Object> settings) { + + } + + @Override + public void validateMapping(MappingLookup lookup) { + + } + + @Override + public void validateAlias(String indexRouting, String searchRouting) { + + } + + @Override + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + + } + + @Override + public CompressedXContent getDefaultMapping() { + return null; + } + + @Override + public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { + return null; + } + + @Override + public IdFieldMapper idFieldMapperWithoutFieldData() { + return null; + } + + @Override + public TimestampBounds getTimestampBound(IndexMetadata indexMetadata) { + return null; + } + + @Override + public MetadataFieldMapper timeSeriesIdFieldMapper() { + return null; + } + + @Override + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + return null; + } + + @Override + public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { + return null; + } + + @Override + public boolean shouldValidateTimestamp() { + return false; + } + + @Override + public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { + + } + + @Override + public boolean isSyntheticSourceEnabled() { + return false; + } }; protected static String tsdbMode() { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index aa92025f32428..dd12ceb62d420 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Booleans; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; @@ -595,6 +596,18 @@ public Iterator> settings() { Property.ServerlessPublic ); + private static final Boolean LOGS_MODE_MODE_FEATURE_FLAG_ENABLED; + + static { + final String property = System.getProperty("es.index_mode_logs_feature_flag_enabled"); + LOGS_MODE_MODE_FEATURE_FLAG_ENABLED = Booleans.parseBoolean(property, false); + } + public static final Setting LOGS_INDEX_MODE_ENABLED = Setting.boolSetting( + "index.mode.logs.enabled", + LOGS_MODE_MODE_FEATURE_FLAG_ENABLED, + Setting.Property.NodeScope + ); + /** * in time series mode, the end time of the index, timestamp must smaller than start_time */ @@ -825,6 +838,17 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this(indexMetadata, nodeSettings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); } + /** + * Determines the index mode depending on settings. The 'logs' mode is only allowed if explicitly enabled by means of + * 'index.mode.logs.enabled'. If not explicitly enabled, 'standard' index mode is used in place of 'logs'. + */ + private static IndexMode indexMode(final Settings nodeSettings, final IndexScopedSettings indexScopedSettings) { + if (IndexMode.LOGS == indexScopedSettings.get(MODE) && LOGS_INDEX_MODE_ENABLED.get(nodeSettings) == false) { + return IndexMode.STANDARD; + } + return indexScopedSettings.get(MODE); + } + /** * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata * while index level settings will overwrite node settings. @@ -842,7 +866,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti nodeName = Node.NODE_NAME_SETTING.get(settings); this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); - mode = scopedSettings.get(MODE); + mode = indexMode(nodeSettings, scopedSettings); this.timestampBounds = mode.getTimestampBound(indexMetadata); if (timestampBounds != null) { scopedSettings.addSettingsUpdateConsumer(IndexSettings.TIME_SERIES_END_TIME, endTime -> { diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 28e435b852af1..7181015203aff 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -35,6 +35,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.index.IndexSettings.LOGS_INDEX_MODE_ENABLED; + public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); @@ -61,6 +63,8 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; + private final boolean logsIndexModeEnabled; + public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream @@ -121,6 +125,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); + this.logsIndexModeEnabled = LOGS_INDEX_MODE_ENABLED.get(nodeSettings); } @Override @@ -167,7 +172,7 @@ protected List getLifecyclePolicies() { static { final Map componentTemplates = new HashMap<>(); - List configs = List.of( + for (IndexTemplateConfig config : List.of( new IndexTemplateConfig( DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, "/data-streams@mappings.json", @@ -238,17 +243,7 @@ protected List getLifecyclePolicies() { TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES ) - ); - configs.add( - new IndexTemplateConfig( - LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs@settings-logsdb.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES - ) - ); - for (IndexTemplateConfig config : configs) { + )) { try { componentTemplates.put( config.getTemplateName(), @@ -263,6 +258,32 @@ protected List getLifecyclePolicies() { @Override protected Map getComponentTemplateConfigs() { + if (logsIndexModeEnabled) { + final IndexTemplateConfig override = new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs@settings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ); + try { + COMPONENT_TEMPLATE_CONFIGS.replace( + override.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, override.loadBytes())) + ); + logger.info("Overriding component template [" + override.getTemplateName() + "] with [" + override.getFileName() + "]"); + } catch (IOException e) { + logger.error( + "Error while overriding template [" + + override.getTemplateName() + + "] with [" + + override.getFileName() + + "]. Error [" + + e.getMessage() + + "]" + ); + } + } return COMPONENT_TEMPLATE_CONFIGS; } From c9adc0c44ecf5cdcec6aa44506401868072d7657 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 May 2024 16:20:58 +0200 Subject: [PATCH 03/54] fix: mode mode --- .../main/java/org/elasticsearch/index/IndexSettings.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index dd12ceb62d420..c6e66f0ba9278 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -596,15 +596,15 @@ public Iterator> settings() { Property.ServerlessPublic ); - private static final Boolean LOGS_MODE_MODE_FEATURE_FLAG_ENABLED; + private static final Boolean LOGS_MODE_FEATURE_FLAG_ENABLED; static { final String property = System.getProperty("es.index_mode_logs_feature_flag_enabled"); - LOGS_MODE_MODE_FEATURE_FLAG_ENABLED = Booleans.parseBoolean(property, false); + LOGS_MODE_FEATURE_FLAG_ENABLED = Booleans.parseBoolean(property, false); } public static final Setting LOGS_INDEX_MODE_ENABLED = Setting.boolSetting( "index.mode.logs.enabled", - LOGS_MODE_MODE_FEATURE_FLAG_ENABLED, + LOGS_MODE_FEATURE_FLAG_ENABLED, Setting.Property.NodeScope ); From 9a05971670f81c07553a3c9bf9a706b454f2d8eb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 May 2024 16:22:44 +0200 Subject: [PATCH 04/54] fix: remove unused code --- .../main/java/org/elasticsearch/test/cluster/FeatureFlag.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 7ff978fcbf6ef..d555337f467ae 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -17,8 +17,7 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), - SEMANTIC_TEXT_ENABLED("es.semantic_text_feature_flag_enabled=true", Version.fromString("8.15.0"), null), - LOGS_INDEX_MODE("es.logs_index_mode_feature_flag_registered=true", Version.fromString("8.15.0"), null); + SEMANTIC_TEXT_ENABLED("es.semantic_text_feature_flag_enabled=true", Version.fromString("8.15.0"), null); public final String systemProperty; public final Version from; From db785c330f933b8e2db30942ab18bf6dceb98963 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 6 Jun 2024 19:57:41 +0200 Subject: [PATCH 05/54] fix: use a FeatureFlag to determine the default setting value --- .../java/org/elasticsearch/index/IndexSettings.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index c6e66f0ba9278..c498bae498ef9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -23,7 +23,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.Booleans; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; @@ -596,15 +596,14 @@ public Iterator> settings() { Property.ServerlessPublic ); - private static final Boolean LOGS_MODE_FEATURE_FLAG_ENABLED; + private static final FeatureFlag LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG = new FeatureFlag("logs_index_mode_enabled"); - static { - final String property = System.getProperty("es.index_mode_logs_feature_flag_enabled"); - LOGS_MODE_FEATURE_FLAG_ENABLED = Booleans.parseBoolean(property, false); - } + /** + * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' + */ public static final Setting LOGS_INDEX_MODE_ENABLED = Setting.boolSetting( "index.mode.logs.enabled", - LOGS_MODE_FEATURE_FLAG_ENABLED, + LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG.isEnabled(), Setting.Property.NodeScope ); From fd8000fea530c814cb202c60fc806e95924e5317 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 6 Jun 2024 20:21:37 +0200 Subject: [PATCH 06/54] fix: missing json file extension --- .../resources/{logs@settings-logsdb => logs@settings-logsdb.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugin/core/template-resources/src/main/resources/{logs@settings-logsdb => logs@settings-logsdb.json} (100%) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json similarity index 100% rename from x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb rename to x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json From f2b4aed8b2bf59ecb974c874a5f867ecd8ec45e9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 6 Jun 2024 20:27:33 +0200 Subject: [PATCH 07/54] fix: unable to replace immutable map entry --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 7181015203aff..feb9640e06bcc 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -266,8 +266,9 @@ protected Map getComponentTemplateConfigs() { TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES ); + final Map updatedComponentTemplates = new HashMap<>(COMPONENT_TEMPLATE_CONFIGS); try { - COMPONENT_TEMPLATE_CONFIGS.replace( + updatedComponentTemplates.replace( override.getTemplateName(), ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, override.loadBytes())) ); @@ -283,6 +284,7 @@ protected Map getComponentTemplateConfigs() { + "]" ); } + return Map.copyOf(updatedComponentTemplates); } return COMPONENT_TEMPLATE_CONFIGS; } From 8f45572ae0f85c5f842df25f35220064e79dc5ff Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 10:48:27 +0200 Subject: [PATCH 08/54] test: include a java rest test for index mode with logs --- .../LogsIndexModeSettingRestTest.java | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java new file mode 100644 index 0000000000000..aaf7b0426a7d8 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.Build; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeSettingRestTest extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.INTEG_TEST) + .setting("xpack.security.enabled", "false") + .setting("stack.templates.enabled", "true") + .module("x-pack-stack") + .module("data-streams") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "index_patterns": [ "logs-*-*" ], + "data_stream": {}, + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "hostname": { + "type": "keyword" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + private static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + public void testLogsSettingsIndexMode() throws IOException { + assertOK(getTemplate(client, "logs")); + assertOK(putTemplate(client, "custom-mappings", MAPPINGS)); + assertOK(getTemplate(client, "custom-mappings")); + assertOK(createDataStream(client, "logs-apache-dev")); + final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-apache-dev", 0), "index.mode"); + if (Build.current().isProductionRelease()) { + assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); + } else { + assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + } + } + + private static Response putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + templateName); + request.setJsonEntity(mappings); + return client.performRequest(request); + } + + private static Response getTemplate(final RestClient client, final String templateName) throws IOException { + return client.performRequest(new Request("GET", "/_index_template/" + templateName)); + } + + private static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); + } + + @SuppressWarnings("unchecked") + private static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.get(backingIndex).get("index_name"); + } + + @SuppressWarnings("unchecked") + private static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); + final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) + .get("settings"); + return settings.get(setting); + } +} From 93916bcb639d1bbf0521d2a6075f74a1122f70aa Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 10:48:52 +0200 Subject: [PATCH 09/54] fix: error if replacing component template failes --- .../xpack/stack/StackTemplateRegistry.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index feb9640e06bcc..8a32a9e4a055a 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -268,11 +268,19 @@ protected Map getComponentTemplateConfigs() { ); final Map updatedComponentTemplates = new HashMap<>(COMPONENT_TEMPLATE_CONFIGS); try { - updatedComponentTemplates.replace( + final ComponentTemplate originalComponentTemplate = updatedComponentTemplates.replace( override.getTemplateName(), ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, override.loadBytes())) ); - logger.info("Overriding component template [" + override.getTemplateName() + "] with [" + override.getFileName() + "]"); + if (originalComponentTemplate == null) { + logger.error( + "Failure while overriding the original [" + + override.getTemplateName() + + "] with [" + + override.getFileName() + + "]. Using default component template." + ); + } } catch (IOException e) { logger.error( "Error while overriding template [" From 04ced4d6fff76fbf8b310931a2649f10e6fb49ae Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 11:00:34 +0200 Subject: [PATCH 10/54] fix: rename class to match naming conventions --- ...SettingRestTest.java => LogsIndexModeSettingRestTestIT.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/{LogsIndexModeSettingRestTest.java => LogsIndexModeSettingRestTestIT.java} (98%) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java similarity index 98% rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java rename to modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java index aaf7b0426a7d8..d9f857d37e32f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTest.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java @@ -26,7 +26,7 @@ import static org.hamcrest.Matchers.equalTo; -public class LogsIndexModeSettingRestTest extends ESRestTestCase { +public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() From ded8123d2cea35c7837871766f6c1ba475639617 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 11:22:08 +0200 Subject: [PATCH 11/54] gradlew: add missing stack plugin dependency --- modules/data-streams/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index a0375c61d7c29..80fbae96676c4 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -21,6 +21,7 @@ restResources { dependencies { testImplementation project(path: ':test:test-clusters') internalClusterTestImplementation project(":modules:mapper-extras") + clusterModules project(':x-pack:plugin:stack') } tasks.named('yamlRestTest') { From 6ef65cdf08cfe794d095d9233ff122fcc2dc50c9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 12:04:57 +0200 Subject: [PATCH 12/54] fix: include username and password for authentication --- .../LogsIndexModeSettingRestTestIT.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java index d9f857d37e32f..bcd23dec97142 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java @@ -8,35 +8,58 @@ package org.elasticsearch.datastreams; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.Before; import org.junit.ClassRule; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.hamcrest.Matchers.equalTo; public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { + private static final String USERNAME = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin")); + private static final String PASSWORD = Objects.requireNonNull( + System.getProperty("tests.rest.cluster.password", "x-pack-test-password") + ); + @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.INTEG_TEST) .setting("xpack.security.enabled", "false") - .setting("stack.templates.enabled", "true") .module("x-pack-stack") .module("data-streams") + .user(USERNAME, PASSWORD) .build(); + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USERNAME, new SecureString(PASSWORD.toCharArray())); + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); From 131610eae6d425bef304bc8e3ce72e4b9a561aea Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 12:13:40 +0200 Subject: [PATCH 13/54] fix: user and pass not required --- .../LogsIndexModeSettingRestTestIT.java | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java index bcd23dec97142..bc81c39ee7264 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java @@ -8,58 +8,32 @@ package org.elasticsearch.datastreams; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.Before; import org.junit.ClassRule; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.hamcrest.Matchers.equalTo; public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { - private static final String USERNAME = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username", "test_admin")); - private static final String PASSWORD = Objects.requireNonNull( - System.getProperty("tests.rest.cluster.password", "x-pack-test-password") - ); - @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.INTEG_TEST) - .setting("xpack.security.enabled", "false") .module("x-pack-stack") .module("data-streams") - .user(USERNAME, PASSWORD) + .setting("stack.templates.enabled", "true") .build(); - @ParametersFactory - public static Iterable parameters() throws Exception { - return ESClientYamlSuiteTestCase.createParameters(); - } - - @Override - protected Settings restClientSettings() { - String token = basicAuthHeaderValue(USERNAME, new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); From 23147db31cc9b914cbc6da8bed907792d1bfaa69 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:19:40 +0200 Subject: [PATCH 14/54] Update docs/changelog/109025.yaml --- docs/changelog/109025.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/changelog/109025.yaml diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml new file mode 100644 index 0000000000000..622af0977249d --- /dev/null +++ b/docs/changelog/109025.yaml @@ -0,0 +1,7 @@ +pr: 109025 +summary: Introduce a feature flag and a setting controlling the activation of the + `logs` index mode +area: Logs +type: feature +issues: + - 108762 From fa81d81f48eb3a86cca40fe88f5f1182876aa1a0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 12:30:39 +0200 Subject: [PATCH 15/54] nit: remove unnecessary code --- .../datastreams/LogsIndexModeSettingRestTestIT.java | 4 +--- server/build.gradle | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java index bc81c39ee7264..f094ccd169743 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java @@ -75,7 +75,7 @@ public void setup() throws Exception { private static void waitForLogs(RestClient client) throws Exception { assertBusy(() -> { try { - Request request = new Request("GET", "_index_template/logs"); + final Request request = new Request("GET", "_index_template/logs"); assertOK(client.performRequest(request)); } catch (ResponseException e) { fail(e.getMessage()); @@ -84,9 +84,7 @@ private static void waitForLogs(RestClient client) throws Exception { } public void testLogsSettingsIndexMode() throws IOException { - assertOK(getTemplate(client, "logs")); assertOK(putTemplate(client, "custom-mappings", MAPPINGS)); - assertOK(getTemplate(client, "custom-mappings")); assertOK(createDataStream(client, "logs-apache-dev")); final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-apache-dev", 0), "index.mode"); if (Build.current().isProductionRelease()) { diff --git a/server/build.gradle b/server/build.gradle index f9c4128e74a44..5831930421c60 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -145,12 +145,10 @@ if (BuildParams.isSnapshotBuild() == false) { tasks.named("test").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' - systemProperty 'es.index_mode_logs_feature_flag_enabled', 'true' } tasks.named("internalClusterTest").configure { systemProperty 'es.index_mode_feature_flag_registered', 'true' systemProperty 'es.failure_store_feature_flag_enabled', 'true' - systemProperty 'es.index_mode_logs_feature_flag_enabled', 'true' } } From 35601b82e0ebfed47e76871382ba639a2e6ff122 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 12:42:43 +0200 Subject: [PATCH 16/54] fix: disable security and use default distro --- .../datastreams/LogsIndexModeSettingRestTestIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java index f094ccd169743..730a1d2aaf566 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; import org.junit.ClassRule; @@ -29,9 +30,11 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) .module("x-pack-stack") .module("data-streams") .setting("stack.templates.enabled", "true") + .setting("xpack.security.enabled", "false") .build(); @Override From f5bcb0faba7e1e7de59720e7c59f4988946c781d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 16:34:42 +0200 Subject: [PATCH 17/54] fix: component template map and cluster modules --- modules/data-streams/build.gradle | 1 - x-pack/plugin/stack/build.gradle | 16 +++ .../LogsIndexModeSettingRestTestIT.java | 21 +++- .../xpack/stack/StackTemplateRegistry.java | 106 +++++++++++++----- 4 files changed, 111 insertions(+), 33 deletions(-) rename {modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams => x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack}/LogsIndexModeSettingRestTestIT.java (90%) diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index 80fbae96676c4..a0375c61d7c29 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -21,7 +21,6 @@ restResources { dependencies { testImplementation project(path: ':test:test-clusters') internalClusterTestImplementation project(":modules:mapper-extras") - clusterModules project(':x-pack:plugin:stack') } tasks.named('yamlRestTest') { diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index c4b950ad9cb59..188f8da1c5e34 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -21,6 +21,22 @@ dependencies { javaRestTestImplementation project(path: ':x-pack:plugin:stack') clusterModules project(':modules:mapper-extras') clusterModules project(xpackModule('wildcard')) + clusterModules project(':modules:data-streams') + clusterModules project(':modules:ingest-common') + clusterModules project(':modules:ingest-geoip') + clusterModules project(':modules:ingest-user-agent') + clusterModules project(':modules:lang-mustache') + clusterModules project(xpackModule('analytics')) + clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(xpackModule('mapper-counted-keyword')) + clusterModules project(xpackModule('stack')) + clusterModules project(xpackModule('apm-data')) + clusterModules project(xpackModule('mapper-aggregate-metric')) +} + +tasks.named('javaRestTest') { + usesDefaultDistribution() } // These tests are only invoked direclty as part of a dedicated build job diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java similarity index 90% rename from modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java rename to x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index 730a1d2aaf566..d3401b3ae16f6 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.datastreams; +package org.elasticsearch.xpack.stack; import org.elasticsearch.Build; import org.elasticsearch.client.Request; @@ -15,7 +15,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; import org.junit.ClassRule; @@ -30,11 +29,21 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("x-pack-stack") + .module("constant-keyword") + .module("counted-keyword") .module("data-streams") - .setting("stack.templates.enabled", "true") - .setting("xpack.security.enabled", "false") + .module("ingest-common") + .module("ingest-geoip") + .module("ingest-user-agent") + .module("lang-mustache") + .module("mapper-extras") + .module("wildcard") + .module("x-pack-analytics") + .module("x-pack-apm-data") + .module("x-pack-aggregate-metric") + .module("x-pack-ilm") + .module("x-pack-stack") + .setting("ingest.geoip.downloader.enabled", "false") .build(); @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 8a32a9e4a055a..a332be30432da 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -259,42 +259,96 @@ protected List getLifecyclePolicies() { @Override protected Map getComponentTemplateConfigs() { if (logsIndexModeEnabled) { - final IndexTemplateConfig override = new IndexTemplateConfig( + return getLogsIndexModeComponentTemplates(); + } + return COMPONENT_TEMPLATE_CONFIGS; + } + + private static Map getLogsIndexModeComponentTemplates() { + final Map updatedTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/data-streams@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/logs@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + ECS_DYNAMIC_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/ecs@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + // NOTE: template including index.mode = 'logs' + new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, "/logs@settings-logsdb.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES - ); - final Map updatedComponentTemplates = new HashMap<>(COMPONENT_TEMPLATE_CONFIGS); + ), + new IndexTemplateConfig( + METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_TSDB_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@tsdb-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, + "/kibana-reporting@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ) + )) { try { - final ComponentTemplate originalComponentTemplate = updatedComponentTemplates.replace( - override.getTemplateName(), - ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, override.loadBytes())) + updatedTemplates.put( + config.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) ); - if (originalComponentTemplate == null) { - logger.error( - "Failure while overriding the original [" - + override.getTemplateName() - + "] with [" - + override.getFileName() - + "]. Using default component template." - ); - } } catch (IOException e) { - logger.error( - "Error while overriding template [" - + override.getTemplateName() - + "] with [" - + override.getFileName() - + "]. Error [" - + e.getMessage() - + "]" - ); + throw new AssertionError(e); } - return Map.copyOf(updatedComponentTemplates); } - return COMPONENT_TEMPLATE_CONFIGS; + return updatedTemplates; } private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( From 7852b7af39447a41b14dc20c4245b616789891ab Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 16:47:29 +0200 Subject: [PATCH 18/54] fix: license header --- .../stack/LogsIndexModeSettingRestTestIT.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index d3401b3ae16f6..aa29c2f38a6b6 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -1,13 +1,31 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.stack; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; From ba39898c455f99c58627d6ad091bb6fa03431d79 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 16:55:12 +0200 Subject: [PATCH 19/54] fix: create another static map --- .../xpack/stack/StackTemplateRegistry.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index a332be30432da..c2ba0990d682a 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -169,6 +169,7 @@ protected List getLifecyclePolicies() { } private static final Map COMPONENT_TEMPLATE_CONFIGS; + private static final Map UPDATED_COMPONENT_TEMPLATE_CONFIGS; static { final Map componentTemplates = new HashMap<>(); @@ -254,18 +255,8 @@ protected List getLifecyclePolicies() { } } COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates); - } - - @Override - protected Map getComponentTemplateConfigs() { - if (logsIndexModeEnabled) { - return getLogsIndexModeComponentTemplates(); - } - return COMPONENT_TEMPLATE_CONFIGS; - } - private static Map getLogsIndexModeComponentTemplates() { - final Map updatedTemplates = new HashMap<>(); + final Map updatedComponentTemplates = new HashMap<>(); for (IndexTemplateConfig config : List.of( new IndexTemplateConfig( DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, @@ -288,7 +279,6 @@ private static Map getLogsIndexModeComponentTemplates TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES ), - // NOTE: template including index.mode = 'logs' new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, "/logs@settings-logsdb.json", @@ -340,7 +330,7 @@ private static Map getLogsIndexModeComponentTemplates ) )) { try { - updatedTemplates.put( + updatedComponentTemplates.put( config.getTemplateName(), ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) ); @@ -348,7 +338,15 @@ private static Map getLogsIndexModeComponentTemplates throw new AssertionError(e); } } - return updatedTemplates; + UPDATED_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(updatedComponentTemplates); + } + + @Override + protected Map getComponentTemplateConfigs() { + if (logsIndexModeEnabled) { + return UPDATED_COMPONENT_TEMPLATE_CONFIGS; + } + return COMPONENT_TEMPLATE_CONFIGS; } private static final Map COMPOSABLE_INDEX_TEMPLATE_CONFIGS = parseComposableTemplates( From 40b0806d2e15ed838f730751ba51bdc455cebcee Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 17:07:56 +0200 Subject: [PATCH 20/54] nit: rename map --- .../xpack/stack/StackTemplateRegistry.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index c2ba0990d682a..41186bc13f20f 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -169,7 +169,7 @@ protected List getLifecyclePolicies() { } private static final Map COMPONENT_TEMPLATE_CONFIGS; - private static final Map UPDATED_COMPONENT_TEMPLATE_CONFIGS; + private static final Map LOGSDB_COMPONENT_TEMPLATE_CONFIGS; static { final Map componentTemplates = new HashMap<>(); @@ -256,7 +256,7 @@ protected List getLifecyclePolicies() { } COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates); - final Map updatedComponentTemplates = new HashMap<>(); + final Map logsdbComponentTemplates = new HashMap<>(); for (IndexTemplateConfig config : List.of( new IndexTemplateConfig( DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, @@ -330,7 +330,7 @@ protected List getLifecyclePolicies() { ) )) { try { - updatedComponentTemplates.put( + logsdbComponentTemplates.put( config.getTemplateName(), ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) ); @@ -338,13 +338,13 @@ protected List getLifecyclePolicies() { throw new AssertionError(e); } } - UPDATED_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(updatedComponentTemplates); + LOGSDB_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(logsdbComponentTemplates); } @Override protected Map getComponentTemplateConfigs() { if (logsIndexModeEnabled) { - return UPDATED_COMPONENT_TEMPLATE_CONFIGS; + return LOGSDB_COMPONENT_TEMPLATE_CONFIGS; } return COMPONENT_TEMPLATE_CONFIGS; } From f72f861f8eb3e8c1b6d63390abd19398a440549a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 17:14:26 +0200 Subject: [PATCH 21/54] fix: do not agte usage of logs index.mode with the setting --- .../java/org/elasticsearch/index/IndexSettings.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index c498bae498ef9..74bc078912f17 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -837,17 +837,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this(indexMetadata, nodeSettings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS); } - /** - * Determines the index mode depending on settings. The 'logs' mode is only allowed if explicitly enabled by means of - * 'index.mode.logs.enabled'. If not explicitly enabled, 'standard' index mode is used in place of 'logs'. - */ - private static IndexMode indexMode(final Settings nodeSettings, final IndexScopedSettings indexScopedSettings) { - if (IndexMode.LOGS == indexScopedSettings.get(MODE) && LOGS_INDEX_MODE_ENABLED.get(nodeSettings) == false) { - return IndexMode.STANDARD; - } - return indexScopedSettings.get(MODE); - } - /** * Creates a new {@link IndexSettings} instance. The given node settings will be merged with the settings in the metadata * while index level settings will overwrite node settings. @@ -865,7 +854,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti nodeName = Node.NODE_NAME_SETTING.get(settings); this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); - mode = indexMode(nodeSettings, scopedSettings); + mode = scopedSettings.get(MODE); this.timestampBounds = mode.getTimestampBound(indexMetadata); if (timestampBounds != null) { scopedSettings.addSettingsUpdateConsumer(IndexSettings.TIME_SERIES_END_TIME, endTime -> { From bf0669f1e1d19814ff576a35bb6944ba4556eed4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 17:21:10 +0200 Subject: [PATCH 22/54] fix: rename and move logsdb index mode setting --- .../org/elasticsearch/index/IndexSettings.java | 12 ------------ .../xpack/stack/StackTemplateRegistry.java | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 74bc078912f17..aa92025f32428 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; @@ -596,17 +595,6 @@ public Iterator> settings() { Property.ServerlessPublic ); - private static final FeatureFlag LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG = new FeatureFlag("logs_index_mode_enabled"); - - /** - * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' - */ - public static final Setting LOGS_INDEX_MODE_ENABLED = Setting.boolSetting( - "index.mode.logs.enabled", - LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG.isEnabled(), - Setting.Property.NodeScope - ); - /** * in time series mode, the end time of the index, timestamp must smaller than start_time */ diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 41186bc13f20f..c137a8714b708 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.threadpool.ThreadPool; @@ -35,7 +36,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.IndexSettings.LOGS_INDEX_MODE_ENABLED; +import static org.elasticsearch.index.IndexSettings.CLUSTER_LOGSDB_ENABLED_SETTING; public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); @@ -59,6 +60,17 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); + private static final FeatureFlag LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG = new FeatureFlag("logs_index_mode_enabled"); + + /** + * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' + */ + public static final Setting CLUSTER_LOGSDB_ENABLED_SETTING = Setting.boolSetting( + "cluster.logsdb.enabled", + LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG.isEnabled(), + Setting.Property.NodeScope + ); + private final ClusterService clusterService; private final FeatureService featureService; private volatile boolean stackTemplateEnabled; @@ -125,7 +137,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); - this.logsIndexModeEnabled = LOGS_INDEX_MODE_ENABLED.get(nodeSettings); + this.logsIndexModeEnabled = CLUSTER_LOGSDB_ENABLED_SETTING.get(nodeSettings); } @Override From 31066c574eebe91c54fe5b5d666689997c6ae1e7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 17:23:33 +0200 Subject: [PATCH 23/54] nit: rename variable --- .../elasticsearch/xpack/stack/StackTemplateRegistry.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index c137a8714b708..679808297b4d6 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -75,7 +75,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private final boolean logsIndexModeEnabled; + private final boolean logsIndexModeTemplateEnabled; public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); @@ -137,7 +137,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); - this.logsIndexModeEnabled = CLUSTER_LOGSDB_ENABLED_SETTING.get(nodeSettings); + this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED_SETTING.get(nodeSettings); } @Override @@ -355,7 +355,7 @@ protected List getLifecyclePolicies() { @Override protected Map getComponentTemplateConfigs() { - if (logsIndexModeEnabled) { + if (logsIndexModeTemplateEnabled) { return LOGSDB_COMPONENT_TEMPLATE_CONFIGS; } return COMPONENT_TEMPLATE_CONFIGS; From fb283a6cdbb47d4988dcae02a22b98e661e3f30c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 7 Jun 2024 17:25:09 +0200 Subject: [PATCH 24/54] fix: unused import --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 679808297b4d6..de19a97e78c08 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -36,8 +36,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.IndexSettings.CLUSTER_LOGSDB_ENABLED_SETTING; - public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); From 93c8b12fc2e057b01a5e3bf515b60e784d9fe590 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Sat, 8 Jun 2024 10:06:19 +0200 Subject: [PATCH 25/54] fix: trial license and no security --- x-pack/plugin/stack/build.gradle | 1 - .../xpack/stack/LogsIndexModeSettingRestTestIT.java | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 188f8da1c5e34..a4eb61a254c91 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -16,7 +16,6 @@ base { dependencies { compileOnly project(path: xpackModule('core')) - testImplementation project(':modules:data-streams') javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') clusterModules project(':modules:mapper-extras') diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index aa29c2f38a6b6..497f314c6e410 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -33,6 +33,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; import org.junit.ClassRule; @@ -47,6 +48,7 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) .module("constant-keyword") .module("counted-keyword") .module("data-streams") @@ -62,6 +64,8 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { .module("x-pack-ilm") .module("x-pack-stack") .setting("ingest.geoip.downloader.enabled", "false") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") .build(); @Override From f32ad99a11db14e1191b54474d20598f8f31fdfe Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Sun, 9 Jun 2024 13:01:35 +0200 Subject: [PATCH 26/54] fix: remove ingest-common module --- x-pack/plugin/stack/build.gradle | 1 - .../xpack/stack/LogsIndexModeSettingRestTestIT.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index a4eb61a254c91..30663acb783e7 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -21,7 +21,6 @@ dependencies { clusterModules project(':modules:mapper-extras') clusterModules project(xpackModule('wildcard')) clusterModules project(':modules:data-streams') - clusterModules project(':modules:ingest-common') clusterModules project(':modules:ingest-geoip') clusterModules project(':modules:ingest-user-agent') clusterModules project(':modules:lang-mustache') diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index 497f314c6e410..c039babe5c44c 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -52,7 +52,6 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { .module("constant-keyword") .module("counted-keyword") .module("data-streams") - .module("ingest-common") .module("ingest-geoip") .module("ingest-user-agent") .module("lang-mustache") From d72e6327129ab92044a45b64cb97d265069bbdf6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 09:46:25 +0200 Subject: [PATCH 27/54] fix: remove unused modules --- x-pack/plugin/stack/build.gradle | 8 -------- .../xpack/stack/LogsIndexModeSettingRestTestIT.java | 9 --------- 2 files changed, 17 deletions(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 30663acb783e7..2b15ffd53825d 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -19,17 +19,9 @@ dependencies { javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') clusterModules project(':modules:mapper-extras') - clusterModules project(xpackModule('wildcard')) clusterModules project(':modules:data-streams') - clusterModules project(':modules:ingest-geoip') - clusterModules project(':modules:ingest-user-agent') - clusterModules project(':modules:lang-mustache') - clusterModules project(xpackModule('analytics')) - clusterModules project(xpackModule('ilm')) clusterModules project(xpackModule('mapper-constant-keyword')) - clusterModules project(xpackModule('mapper-counted-keyword')) clusterModules project(xpackModule('stack')) - clusterModules project(xpackModule('apm-data')) clusterModules project(xpackModule('mapper-aggregate-metric')) } diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index c039babe5c44c..7c41ba111d851 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -50,19 +50,10 @@ public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .module("constant-keyword") - .module("counted-keyword") .module("data-streams") - .module("ingest-geoip") - .module("ingest-user-agent") - .module("lang-mustache") .module("mapper-extras") - .module("wildcard") - .module("x-pack-analytics") - .module("x-pack-apm-data") .module("x-pack-aggregate-metric") - .module("x-pack-ilm") .module("x-pack-stack") - .setting("ingest.geoip.downloader.enabled", "false") .setting("xpack.security.enabled", "false") .setting("xpack.license.self_generated.type", "trial") .build(); From 465adc20e9f546c0bd4c760b2878e422d9887252 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 09:46:54 +0200 Subject: [PATCH 28/54] fix: use logs@custom to override mappings --- .../stack/LogsIndexModeSettingRestTestIT.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index 7c41ba111d851..51c7730901b8d 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -73,9 +73,6 @@ public void setup() throws Exception { private static final String MAPPINGS = """ { - "index_patterns": [ "logs-*-*" ], - "data_stream": {}, - "priority": 500, "template": { "mappings": { "properties": { @@ -108,9 +105,9 @@ private static void waitForLogs(RestClient client) throws Exception { } public void testLogsSettingsIndexMode() throws IOException { - assertOK(putTemplate(client, "custom-mappings", MAPPINGS)); - assertOK(createDataStream(client, "logs-apache-dev")); - final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-apache-dev", 0), "index.mode"); + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-custom-dev", 0), "index.mode"); if (Build.current().isProductionRelease()) { assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); } else { @@ -118,16 +115,13 @@ public void testLogsSettingsIndexMode() throws IOException { } } - private static Response putTemplate(final RestClient client, final String templateName, final String mappings) throws IOException { - final Request request = new Request("PUT", "/_index_template/" + templateName); + private static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + throws IOException { + final Request request = new Request("PUT", "/_component_template/" + templateName); request.setJsonEntity(mappings); return client.performRequest(request); } - private static Response getTemplate(final RestClient client, final String templateName) throws IOException { - return client.performRequest(new Request("GET", "/_index_template/" + templateName)); - } - private static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); } From 8c0e74763cada326b57aa45ea69b97b753cf99b7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 10:08:54 +0200 Subject: [PATCH 29/54] fix: check no logs index mode in release build --- .../xpack/stack/LogsIndexModeSettingRestTestIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index 51c7730901b8d..44cceab3fb76a 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -35,6 +35,7 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; @@ -109,7 +110,7 @@ public void testLogsSettingsIndexMode() throws IOException { assertOK(createDataStream(client, "logs-custom-dev")); final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-custom-dev", 0), "index.mode"); if (Build.current().isProductionRelease()) { - assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); + assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); } else { assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); } From 754b85f6779bcf0323d5091a835a0477e814cbfc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 10:53:47 +0200 Subject: [PATCH 30/54] fix: include hostname in logs mappings --- .../main/resources/logs@mappings-logsdb.json | 31 +++++++++++++++++++ .../xpack/stack/StackTemplateRegistry.java | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json new file mode 100644 index 0000000000000..08e56f9f07b71 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json @@ -0,0 +1,31 @@ +{ + "template": { + "mappings": { + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "hostname": { + "type": "keyword" + }, + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + }, + "_meta": { + "description": "default mappings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index de19a97e78c08..b9a4afb2772c5 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -277,7 +277,7 @@ protected List getLifecyclePolicies() { ), new IndexTemplateConfig( LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, - "/logs@mappings.json", + "/logs@mappings-logsdb.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES From 7c43f5aec6ef14a21ca08221926ff60082c9cadb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 11:01:15 +0200 Subject: [PATCH 31/54] fix: rely on hotname and timestamp from logs@mappings --- .../xpack/stack/LogsIndexModeSettingRestTestIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java index 44cceab3fb76a..684329fed600f 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java @@ -77,12 +77,6 @@ public void setup() throws Exception { "template": { "mappings": { "properties": { - "@timestamp" : { - "type": "date" - }, - "hostname": { - "type": "keyword" - }, "method": { "type": "keyword" }, From 1b3828c4a4f916edea03751536b7067aa65bdfdc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 11:17:01 +0200 Subject: [PATCH 32/54] fix: use 'host.name' instead of 'hostname' --- .../rest-api-spec/test/logsdb/10_settings.yml | 40 +++++++++---------- .../elasticsearch/index/IndexSortConfig.java | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 128903f4faac8..5e8948b7fdea3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -31,7 +31,7 @@ create logs index: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -48,17 +48,17 @@ create logs index: refresh: true body: - { "index": { } } - - { "@timestamp": "2024-02-12T10:30:00Z", "hostname": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } + - { "@timestamp": "2024-02-12T10:30:00Z", ignored_field_stats: "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:31:00Z", "hostname": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:32:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:33:00Z", "hostname": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:34:00Z", "hostname": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:35:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } - do: @@ -103,7 +103,7 @@ using default timestamp field mapping: number_of_shards: 2 mappings: properties: - hostname: + host.name: type: keyword agent_id: type: keyword @@ -149,7 +149,7 @@ missing hostname field: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[hostname]" } + - match: { error.reason: "unknown index sort field:[host.name]" } --- missing sort field: @@ -177,7 +177,7 @@ missing sort field: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -271,7 +271,7 @@ override sort order settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -319,7 +319,7 @@ override sort missing settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -367,7 +367,7 @@ override sort mode settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -410,12 +410,12 @@ override sort field using nested field type in sorting: number_of_replicas: 0 number_of_shards: 2 sort: - field: [ "hostname", "nested", "@timestamp" ] + field: [ "host.name", "nested", "@timestamp" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -459,7 +459,7 @@ override sort field using nested field type: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -499,12 +499,12 @@ routing path not allowed in logs mode: mode: logs number_of_replicas: 0 number_of_shards: 2 - routing_path: [ "hostname", "agent_id" ] + routing_path: [ "host.name", "agent_id" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -545,7 +545,7 @@ start time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -586,7 +586,7 @@ end time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 74c2c57594e72..f190462d6d1e9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -154,7 +154,7 @@ public IndexSortConfig(IndexSettings indexSettings) { List fields = INDEX_SORT_FIELD_SETTING.get(settings); if (this.indexMode == IndexMode.LOGS && fields.isEmpty()) { - fields = List.of("hostname", DataStream.TIMESTAMP_FIELD_NAME); + fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME); } this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); From 268ddb8ae2bbe2a281c5af6e1a2f6bbc20a57023 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 14:00:44 +0200 Subject: [PATCH 33/54] fix: use host.name in logs@mappings --- .../src/main/resources/logs@mappings-logsdb.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json index 08e56f9f07b71..167efbd3ffaf5 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json @@ -6,7 +6,7 @@ "@timestamp": { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "data_stream.type": { From bd3104265d1d1d8502dccd6c37db0ab6eb1dd52a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 14:17:29 +0200 Subject: [PATCH 34/54] fix: add data-streams to test classpath --- x-pack/plugin/stack/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 2b15ffd53825d..27a6580821355 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -16,6 +16,7 @@ base { dependencies { compileOnly project(path: xpackModule('core')) + testImplementation project(':modules:data-streams') javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') clusterModules project(':modules:mapper-extras') From 305ec4132e82ae89084bc80211b8b436ee32e83e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 14:28:06 +0200 Subject: [PATCH 35/54] fix: missing wildcard xpack module --- x-pack/plugin/stack/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index 27a6580821355..c07fc13638253 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -19,6 +19,7 @@ dependencies { testImplementation project(':modules:data-streams') javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') + clusterModules project(xpackModule('wildcard')) clusterModules project(':modules:mapper-extras') clusterModules project(':modules:data-streams') clusterModules project(xpackModule('mapper-constant-keyword')) From b0be5b4ad47971711c5f748b87ce6861a82a7e0d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 15:40:14 +0200 Subject: [PATCH 36/54] fix: use 'host.name' instead of 'hostname' --- .../datastreams/LogsDataStreamIT.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 8a343ff9cf853..996f686d85bfb 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -62,7 +62,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -86,7 +86,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -110,7 +110,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String LOG_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", @@ -121,7 +121,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String TIME_SERIES_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "ip_address": "%s", @@ -207,7 +207,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -224,7 +224,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut assertDataStreamBackingIndicesModes(dataStreamName, List.of(IndexMode.LOGS, IndexMode.TIME_SERIES, IndexMode.LOGS)); } - public void testInvalidIndexModeTimeSeriesSwitchWithoutROutingPath() throws IOException, ExecutionException, InterruptedException { + public void testInvalidIndexModeTimeSeriesSwitchWithoutRoutingPath() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); @@ -250,7 +250,7 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -270,7 +270,7 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc exception.getCause().getCause().getMessage(), Matchers.equalTo( "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with " - + "a list of dimensions in [time_series_dimensions] and without the [script] parameter. [hostname] was not a dimension." + + "a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a dimension." ) ); } From 06d4ff7cef6c78a2df142a79efdb5cf58c4b15d6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 16:25:53 +0200 Subject: [PATCH 37/54] checkstyle: line too long --- .../java/org/elasticsearch/datastreams/LogsDataStreamIT.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 996f686d85bfb..e79d7725989a8 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -269,8 +269,9 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc assertThat( exception.getCause().getCause().getMessage(), Matchers.equalTo( - "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with " - + "a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a dimension." + "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields " + + " with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a" + + "a dimension." ) ); } From d3b7cc855bb48a588dba9d0bc332194d49574361 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 10 Jun 2024 17:03:03 +0200 Subject: [PATCH 38/54] checkstyle: line too long --- .../java/org/elasticsearch/datastreams/LogsDataStreamIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index e79d7725989a8..f95d9a0b0431f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -270,8 +270,8 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc exception.getCause().getCause().getMessage(), Matchers.equalTo( "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields " - + " with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a" - + "a dimension." + + "with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a " + + "dimension." ) ); } From 5b797fc011086b842374b8c73b5adc6c5d252ae6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jun 2024 10:06:11 +0200 Subject: [PATCH 39/54] test: disable test waiting for ignore_malformed support --- .../java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java | 2 ++ .../java/org/elasticsearch/datastreams/LogsDataStreamIT.java | 1 + 2 files changed, 3 insertions(+) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java index 5fe72c38078ee..be737833f1637 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -66,6 +66,7 @@ public void cleanUp() throws IOException { } @SuppressWarnings("unchecked") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testElasticAgentLogEcsMappings() throws Exception { { Path path = PathUtils.get(Thread.currentThread().getContextClassLoader().getResource("ecs-logs/es-agent-ecs-log.json").toURI()); @@ -126,6 +127,7 @@ public void testElasticAgentLogEcsMappings() throws Exception { } @SuppressWarnings("unchecked") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testGeneralMockupEcsMappings() throws Exception { { indexDoc(client, DATA_STREAM_NAME, """ diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index c2a7a76ab751a..c35d42424d3f8 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -221,6 +221,7 @@ public void testCustomMapping() throws Exception { } @SuppressWarnings("unchecked") + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testLogsDefaultPipeline() throws Exception { { Request request = new Request("POST", "/_component_template/logs@custom"); From a9b7a248f9f2bafab43b2efc42a650268459f905 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jun 2024 10:10:23 +0200 Subject: [PATCH 40/54] fix: synthetic source reconstruction artifacts --- .../datastreams/LogsDataStreamIT.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index c35d42424d3f8..95ee67a8a25e2 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -377,12 +377,21 @@ public void testLogsMessagePipeline() throws Exception { assertThat(((List) fields.get("message")).get(0), is("json")); // successful access to subfields verifies that dot expansion is part of the pipeline - assertThat(source.get("log.level"), is("INFO")); - assertThat(source.get("ecs.version"), is("1.6.0")); - assertThat(source.get("service.name"), is("my-app")); - assertThat(source.get("event.dataset"), is("my-app.RollingFile")); - assertThat(source.get("process.thread.name"), is("main")); - assertThat(source.get("log.logger"), is("root.pkg.MyApp")); + // NOTE: dotted field names are expanded to the corresponding objects when synthetic source + // is used, as it is the case for logs. This will be fixed by having `subobjects: false` by + // defaut. See https://github.com/elastic/elasticsearch/issues/106812 + final Map log = (Map) source.get("log"); + assertThat(log.get("level"), is("INFO")); + final Map ecs = (Map) source.get("ecs"); + assertThat(ecs.get("version"), is("1.6.0")); + final Map service = (Map) source.get("service"); + assertThat(service.get("name"), is("my-app")); + final Map event = (Map) source.get("event"); + assertThat(event.get("dataset"), is("my-app.RollingFile")); + final Map process = (Map) source.get("process"); + final Map thread = (Map) process.get("thread"); + assertThat(thread.get("name"), is("main")); + assertThat(log.get("logger"), is("root.pkg.MyApp")); // _tmp_json_message should be removed by the pipeline assertThat(source.get("_tmp_json_message"), is(nullValue())); } @@ -411,7 +420,7 @@ public void testLogsMessagePipeline() throws Exception { Map source = ((Map>) results.get(0)).get("_source"); // root field parsed from JSON should win - assertThat(source.get("@timestamp"), is("2023-05-10")); + assertThat(source.get("@timestamp"), is("2023-05-10T00:00:00.000Z")); assertThat(source.get("message"), is("{\"@timestamp\":\"2023-05-09T16:48:34.135Z\", \"message\":\"malformed_json\"}}")); assertThat(source.get("_tmp_json_message"), is(nullValue())); } @@ -440,7 +449,7 @@ public void testLogsMessagePipeline() throws Exception { Map source = ((Map>) results.get(0)).get("_source"); Map fields = ((Map>) results.get(0)).get("fields"); - assertThat(source.get("message"), is(42)); + assertThat(source.get("message"), is("42")); assertThat(((List) fields.get("message")).get(0), is("42")); } } From b9b28ad95b7a85cfafd4ef74622beacfbce60d19 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 11 Jun 2024 10:11:03 +0200 Subject: [PATCH 41/54] nit: typo --- .../java/org/elasticsearch/datastreams/LogsDataStreamIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 95ee67a8a25e2..af1da45f7d356 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -379,7 +379,7 @@ public void testLogsMessagePipeline() throws Exception { // successful access to subfields verifies that dot expansion is part of the pipeline // NOTE: dotted field names are expanded to the corresponding objects when synthetic source // is used, as it is the case for logs. This will be fixed by having `subobjects: false` by - // defaut. See https://github.com/elastic/elasticsearch/issues/106812 + // default. See https://github.com/elastic/elasticsearch/issues/106812 final Map log = (Map) source.get("log"); assertThat(log.get("level"), is("INFO")); final Map ecs = (Map) source.get("ecs"); From d06c6be5c370abd6d419ea7d360c56ee6f5026c7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 17:49:29 +0200 Subject: [PATCH 42/54] fix: move logsdb java rest test to data-streams module are remove feature flag --- .../datastreams/EcsLogsDataStreamIT.java | 2 - .../datastreams/LogsDataStreamIT.java | 26 +--- .../LogsIndexModeDisabledRestTestIT.java | 91 ++++++++++++ .../LogsIndexModeEnabledRestTestIT.java | 99 +++++++++++++ .../logsdb/LogsIndexModeRestTestIT.java | 83 +++++++++++ .../stack/LogsIndexModeSettingRestTestIT.java | 140 ------------------ .../xpack/stack/StackPlugin.java | 3 +- .../xpack/stack/StackTemplateRegistry.java | 11 +- 8 files changed, 286 insertions(+), 169 deletions(-) create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java delete mode 100644 x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java index be737833f1637..5fe72c38078ee 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -66,7 +66,6 @@ public void cleanUp() throws IOException { } @SuppressWarnings("unchecked") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testElasticAgentLogEcsMappings() throws Exception { { Path path = PathUtils.get(Thread.currentThread().getContextClassLoader().getResource("ecs-logs/es-agent-ecs-log.json").toURI()); @@ -127,7 +126,6 @@ public void testElasticAgentLogEcsMappings() throws Exception { } @SuppressWarnings("unchecked") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testGeneralMockupEcsMappings() throws Exception { { indexDoc(client, DATA_STREAM_NAME, """ diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index af1da45f7d356..c2a7a76ab751a 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -221,7 +221,6 @@ public void testCustomMapping() throws Exception { } @SuppressWarnings("unchecked") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106483") public void testLogsDefaultPipeline() throws Exception { { Request request = new Request("POST", "/_component_template/logs@custom"); @@ -377,21 +376,12 @@ public void testLogsMessagePipeline() throws Exception { assertThat(((List) fields.get("message")).get(0), is("json")); // successful access to subfields verifies that dot expansion is part of the pipeline - // NOTE: dotted field names are expanded to the corresponding objects when synthetic source - // is used, as it is the case for logs. This will be fixed by having `subobjects: false` by - // default. See https://github.com/elastic/elasticsearch/issues/106812 - final Map log = (Map) source.get("log"); - assertThat(log.get("level"), is("INFO")); - final Map ecs = (Map) source.get("ecs"); - assertThat(ecs.get("version"), is("1.6.0")); - final Map service = (Map) source.get("service"); - assertThat(service.get("name"), is("my-app")); - final Map event = (Map) source.get("event"); - assertThat(event.get("dataset"), is("my-app.RollingFile")); - final Map process = (Map) source.get("process"); - final Map thread = (Map) process.get("thread"); - assertThat(thread.get("name"), is("main")); - assertThat(log.get("logger"), is("root.pkg.MyApp")); + assertThat(source.get("log.level"), is("INFO")); + assertThat(source.get("ecs.version"), is("1.6.0")); + assertThat(source.get("service.name"), is("my-app")); + assertThat(source.get("event.dataset"), is("my-app.RollingFile")); + assertThat(source.get("process.thread.name"), is("main")); + assertThat(source.get("log.logger"), is("root.pkg.MyApp")); // _tmp_json_message should be removed by the pipeline assertThat(source.get("_tmp_json_message"), is(nullValue())); } @@ -420,7 +410,7 @@ public void testLogsMessagePipeline() throws Exception { Map source = ((Map>) results.get(0)).get("_source"); // root field parsed from JSON should win - assertThat(source.get("@timestamp"), is("2023-05-10T00:00:00.000Z")); + assertThat(source.get("@timestamp"), is("2023-05-10")); assertThat(source.get("message"), is("{\"@timestamp\":\"2023-05-09T16:48:34.135Z\", \"message\":\"malformed_json\"}}")); assertThat(source.get("_tmp_json_message"), is(nullValue())); } @@ -449,7 +439,7 @@ public void testLogsMessagePipeline() throws Exception { Map source = ((Map>) results.get(0)).get("_source"); Map fields = ((Map>) results.get(0)).get("fields"); - assertThat(source.get("message"), is("42")); + assertThat(source.get("message"), is(42)); assertThat(((List) fields.get("message")).get(0), is("42")); } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java new file mode 100644 index 0000000000000..ad12aff20c825 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.datastreams.logsdb; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + public void testLogsSettingsIndexModeDisabled() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java new file mode 100644 index 0000000000000..23637dde3a809 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + private static String BULK_INDEX_REQUEST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + """; + + public void testLogsSettingsIndexModeCreateDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + } + + public void testLogsIndexBulkIndexing() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + assertOK(bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST)); + } + + public void testLogsIndexRollover() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); + assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); + assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java new file mode 100644 index 0000000000000..9a2b444dfe887 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { + protected static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + final Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + throws IOException { + final Request request = new Request("PUT", "/_component_template/" + templateName); + request.setJsonEntity(mappings); + return client.performRequest(request); + } + + protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); + } + + protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); + } + + @SuppressWarnings("unchecked") + protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) + throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.get(backingIndex).get("index_name"); + } + + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); + final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) + .get("settings"); + return settings.get(setting); + } + + protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) + throws IOException { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + bulkRequest.setJsonEntity(bulkSupplier.get()); + bulkRequest.addParameter("refresh", "true"); + return client().performRequest(bulkRequest); + } +} diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java deleted file mode 100644 index 684329fed600f..0000000000000 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/LogsIndexModeSettingRestTestIT.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.stack; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.elasticsearch.Build; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.ClassRule; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class LogsIndexModeSettingRestTestIT extends ESRestTestCase { - - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Before - public void setup() throws Exception { - client = client(); - waitForLogs(client); - } - - private RestClient client; - - private static final String MAPPINGS = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - } - } - } - } - }"""; - - private static void waitForLogs(RestClient client) throws Exception { - assertBusy(() -> { - try { - final Request request = new Request("GET", "_index_template/logs"); - assertOK(client.performRequest(request)); - } catch (ResponseException e) { - fail(e.getMessage()); - } - }); - } - - public void testLogsSettingsIndexMode() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getWriteBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - if (Build.current().isProductionRelease()) { - assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); - } else { - assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); - } - } - - private static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) - throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); - return client.performRequest(request); - } - - private static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { - return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); - } - - @SuppressWarnings("unchecked") - private static String getWriteBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) throws IOException { - final Request request = new Request("GET", "_data_stream/" + dataStreamName); - final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); - final Map dataStream = (Map) dataStreams.get(0); - final List> backingIndices = (List>) dataStream.get("indices"); - return backingIndices.get(backingIndex).get("index_name"); - } - - @SuppressWarnings("unchecked") - private static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { - final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); - final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) - .get("settings"); - return settings.get(setting); - } -} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index 2577cf28f4213..cc127883652af 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -12,7 +12,6 @@ import org.elasticsearch.plugins.Plugin; import java.util.Collection; -import java.util.Collections; import java.util.List; public class StackPlugin extends Plugin implements ActionPlugin { @@ -24,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return Collections.singletonList(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index b9a4afb2772c5..ebe9acd831084 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.threadpool.ThreadPool; @@ -58,14 +57,12 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); - private static final FeatureFlag LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG = new FeatureFlag("logs_index_mode_enabled"); - /** * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' */ - public static final Setting CLUSTER_LOGSDB_ENABLED_SETTING = Setting.boolSetting( + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( "cluster.logsdb.enabled", - LOGS_INDEX_MODE_ENABLED_FEATURE_FLAG.isEnabled(), + false, Setting.Property.NodeScope ); @@ -73,7 +70,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private final boolean logsIndexModeTemplateEnabled; + private volatile boolean logsIndexModeTemplateEnabled; public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); @@ -135,7 +132,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); - this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED_SETTING.get(nodeSettings); + this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED.get(nodeSettings); } @Override From ebebe733becb39b56a4ca200a963c10cd7306f55 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 18:35:42 +0200 Subject: [PATCH 43/54] fix: license headers --- .../LogsIndexModeDisabledRestTestIT.java | 24 +---- .../LogsIndexModeEnabledRestTestIT.java | 99 ------------------- .../logsdb/LogsIndexModeRestTestIT.java | 83 ---------------- 3 files changed, 3 insertions(+), 203 deletions(-) delete mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java delete mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index ad12aff20c825..f042ec7e31083 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -1,31 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.datastreams.logsdb; -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java deleted file mode 100644 index 23637dde3a809..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.RestClient; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.ClassRule; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { - - @ClassRule() - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("constant-keyword") - .module("data-streams") - .module("mapper-extras") - .module("x-pack-aggregate-metric") - .module("x-pack-stack") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") - .setting("cluster.logsdb.enabled", "true") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Before - public void setup() throws Exception { - client = client(); - waitForLogs(client); - } - - private RestClient client; - - private static final String MAPPINGS = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - } - } - } - } - }"""; - - private static String BULK_INDEX_REQUEST = """ - { "create": {}} - { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } - { "create": {}} - { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } - """; - - public void testLogsSettingsIndexModeCreateDataStream() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); - assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); - } - - public void testLogsIndexBulkIndexing() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - assertOK(bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST)); - } - - public void testLogsIndexRollover() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); - assertOK(rolloverDataStream(client, "logs-custom-dev")); - final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); - assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); - assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java deleted file mode 100644 index 9a2b444dfe887..0000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.datastreams.logsdb; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.test.rest.ESRestTestCase; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { - protected static void waitForLogs(RestClient client) throws Exception { - assertBusy(() -> { - try { - final Request request = new Request("GET", "_index_template/logs"); - assertOK(client.performRequest(request)); - } catch (ResponseException e) { - fail(e.getMessage()); - } - }); - } - - protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) - throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); - return client.performRequest(request); - } - - protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { - return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); - } - - protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { - return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); - } - - @SuppressWarnings("unchecked") - protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) - throws IOException { - final Request request = new Request("GET", "_data_stream/" + dataStreamName); - final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); - final Map dataStream = (Map) dataStreams.get(0); - final List> backingIndices = (List>) dataStream.get("indices"); - return backingIndices.get(backingIndex).get("index_name"); - } - - @SuppressWarnings("unchecked") - protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { - final Request request = new Request("GET", "_data_stream/" + dataStreamName); - final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); - final Map dataStream = (Map) dataStreams.get(0); - final List> backingIndices = (List>) dataStream.get("indices"); - return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); - } - - @SuppressWarnings("unchecked") - protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { - final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); - final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) - .get("settings"); - return settings.get(setting); - } - - protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) - throws IOException { - var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); - bulkRequest.setJsonEntity(bulkSupplier.get()); - bulkRequest.addParameter("refresh", "true"); - return client().performRequest(bulkRequest); - } -} From 2e72371b5d73b7cf655d1075b17b97fb421085bf Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 18:39:34 +0200 Subject: [PATCH 44/54] fix: remove unnecessary modules after moving tests to data-streams --- x-pack/plugin/stack/build.gradle | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index c07fc13638253..c4b950ad9cb59 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -19,16 +19,8 @@ dependencies { testImplementation project(':modules:data-streams') javaRestTestImplementation(testArtifact(project(xpackModule('core')))) javaRestTestImplementation project(path: ':x-pack:plugin:stack') - clusterModules project(xpackModule('wildcard')) clusterModules project(':modules:mapper-extras') - clusterModules project(':modules:data-streams') - clusterModules project(xpackModule('mapper-constant-keyword')) - clusterModules project(xpackModule('stack')) - clusterModules project(xpackModule('mapper-aggregate-metric')) -} - -tasks.named('javaRestTest') { - usesDefaultDistribution() + clusterModules project(xpackModule('wildcard')) } // These tests are only invoked direclty as part of a dedicated build job From 1f21f7665d1050e8bbd7d0258eacba7769619812 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 19:16:52 +0200 Subject: [PATCH 45/54] fix: missing test files --- .../LogsIndexModeEnabledRestTestIT.java | 109 ++++++++++++++++++ .../logsdb/LogsIndexModeRestTestIT.java | 86 ++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java new file mode 100644 index 0000000000000..8cc3441b065ed --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + private static String BULK_INDEX_REQUEST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + """; + + public void testLogsSettingsIndexModeCreateDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + } + + public void testLogsIndexModeSettings() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String sortField = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); + assertThat(sortField, equalTo("bla")); + final String sortOrder = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); + assertThat(sortOrder, equalTo("meh")); + } + + public void testLogsIndexBulkIndexing() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + assertOK(bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST)); + } + + public void testLogsIndexRollover() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); + assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); + assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java new file mode 100644 index 0000000000000..f99d5a6dd70ea --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { + protected static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + final Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + throws IOException { + final Request request = new Request("PUT", "/_component_template/" + templateName); + request.setJsonEntity(mappings); + return client.performRequest(request); + } + + protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); + } + + protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); + } + + @SuppressWarnings("unchecked") + protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) + throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.get(backingIndex).get("index_name"); + } + + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); + final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) + .get("settings"); + + + return settings.get(setting); + } + + protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) + throws IOException { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + bulkRequest.setJsonEntity(bulkSupplier.get()); + bulkRequest.addParameter("refresh", "true"); + return client().performRequest(bulkRequest); + } +} From 399eee2189b280b3bb131929f0bf12e7c70f89ae Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 22:23:08 +0200 Subject: [PATCH 46/54] fix: remove test --- .../logsdb/LogsIndexModeEnabledRestTestIT.java | 9 --------- .../datastreams/logsdb/LogsIndexModeRestTestIT.java | 1 - 2 files changed, 10 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index 8cc3441b065ed..8758e272424c5 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -82,15 +82,6 @@ public void testLogsSettingsIndexModeCreateDataStream() throws IOException { assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); } - public void testLogsIndexModeSettings() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); - assertOK(createDataStream(client, "logs-custom-dev")); - final String sortField = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.field"); - assertThat(sortField, equalTo("bla")); - final String sortOrder = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.sort.order"); - assertThat(sortOrder, equalTo("meh")); - } - public void testLogsIndexBulkIndexing() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java index f99d5a6dd70ea..80a040a83ee17 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -72,7 +72,6 @@ protected static Object getSetting(final RestClient client, final String indexNa final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) .get("settings"); - return settings.get(setting); } From c884c003217bd9f1f858d545f4e06523f23568f7 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 22:26:28 +0200 Subject: [PATCH 47/54] fix: no need for logs@custom --- .../logsdb/LogsIndexModeDisabledRestTestIT.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index f042ec7e31083..dcd2457b88f18 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -47,24 +47,7 @@ public void setup() throws Exception { private RestClient client; - private static final String MAPPINGS = """ - { - "template": { - "mappings": { - "properties": { - "method": { - "type": "keyword" - }, - "message": { - "type": "text" - } - } - } - } - }"""; - public void testLogsSettingsIndexModeDisabled() throws IOException { - assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); From c1a2286ff6c8ecba93bbea61290b3efb7f1ccd4a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 22:37:02 +0200 Subject: [PATCH 48/54] fix: rename test methods --- .../datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index 8758e272424c5..a159d21ebfb53 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -75,20 +75,20 @@ public void setup() throws Exception { { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } """; - public void testLogsSettingsIndexModeCreateDataStream() throws IOException { + public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); } - public void testLogsIndexBulkIndexing() throws IOException { + public void testBulkIndexing() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); assertOK(bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST)); } - public void testLogsIndexRollover() throws IOException { + public void testRolloverDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); From 813de461d1a0e3139d32822d56c77519fad98590 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 12 Jun 2024 23:25:17 +0200 Subject: [PATCH 49/54] fix: hostname to host.name --- .../org/elasticsearch/datastreams/LogsDataStreamRestIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java index c18bcf750242f..d3ec5b29ff5b9 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java @@ -80,7 +80,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -116,7 +116,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -138,7 +138,7 @@ private static void waitForLogs(RestClient client) throws Exception { private static final String DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", From d0090213cae816b364907e81f6709696b3ddac07 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Jun 2024 00:36:44 +0200 Subject: [PATCH 50/54] changelog: update summary --- docs/changelog/109025.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml index 622af0977249d..38d19cab13d30 100644 --- a/docs/changelog/109025.yaml +++ b/docs/changelog/109025.yaml @@ -1,6 +1,5 @@ pr: 109025 -summary: Introduce a feature flag and a setting controlling the activation of the - `logs` index mode +summary: Introduce a setting controlling the activation of the `logs` index mode in logs@settings area: Logs type: feature issues: From 96760bdd62b7ca79da8e87018d4a1bc15a660f55 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Jun 2024 12:30:43 +0200 Subject: [PATCH 51/54] fix: test alternate host mappings --- .../LogsIndexModeEnabledRestTestIT.java | 124 +++++++++++++++++- .../logsdb/LogsIndexModeRestTestIT.java | 2 +- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index a159d21ebfb53..7161dbaab7975 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.datastreams.logsdb; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -64,6 +65,89 @@ public void setup() throws Exception { } }"""; + private static final String ALTERNATE_HOST_MAPPING = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host.cloud_region": { + "type": "keyword" + }, + "host.availability_zone": { + "type": "keyword" + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "subobjects": false, + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + private static String BULK_INDEX_REQUEST = """ { "create": {}} { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } @@ -75,6 +159,18 @@ public void setup() throws Exception { { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } """; + private static String BULK_INDEX_REQUEST_WITH_HOST = + """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } + """; + public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); @@ -85,7 +181,33 @@ public void testCreateDataStream() throws IOException { public void testBulkIndexing() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); - assertOK(bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST)); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithFlatHostProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", ALTERNATE_HOST_MAPPING)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostDefaultSubobjectsProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostSubobjectsFalseProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); } public void testRolloverDataStream() throws IOException { diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java index 80a040a83ee17..ff45096146280 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -80,6 +80,6 @@ protected static Response bulkIndex(final RestClient client, final String dataSt var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); bulkRequest.setJsonEntity(bulkSupplier.get()); bulkRequest.addParameter("refresh", "true"); - return client().performRequest(bulkRequest); + return client.performRequest(bulkRequest); } } From 0ba9e895719d45b82d03c03ab5b62338b0f9eabf Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Jun 2024 12:37:02 +0200 Subject: [PATCH 52/54] fix: use strict mapping --- .../datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index 7161dbaab7975..f5ee10b8afd64 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -121,6 +121,7 @@ public void setup() throws Exception { { "template": { "mappings": { + "dynamic": "strict", "properties": { "method": { "type": "keyword" From f3e109073675ec8aa416c7c09314c8446240c0c5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 13 Jun 2024 12:47:55 +0200 Subject: [PATCH 53/54] checkstyle: line too long --- .../LogsIndexModeEnabledRestTestIT.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index f5ee10b8afd64..832267cebf97c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -160,17 +160,20 @@ public void setup() throws Exception { { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } """; - private static String BULK_INDEX_REQUEST_WITH_HOST = - """ - { "create": {}} - { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } - { "create": {}} - { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } - """; + private static String BULK_INDEX_REQUEST_WITH_HOST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } + """; public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); From 7ce8af0d84837d332f3ea671298b4a7e0bef9387 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 17 Jun 2024 09:39:36 +0200 Subject: [PATCH 54/54] fix: volatile to final --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 1fece772ec11d..34cacbb8956e5 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -70,7 +70,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private volatile boolean logsIndexModeTemplateEnabled; + private final boolean logsIndexModeTemplateEnabled; public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false");