From 3d4e28aee144e46ee0e46aec07c078366f21322d Mon Sep 17 00:00:00 2001 From: Yu Date: Tue, 12 Sep 2017 14:29:10 +0200 Subject: [PATCH] Remove index mapper dynamic settings (#25734) Remove "index.mapper.dynamic" setting for 6.0 (and after) indices, but still keep working for 5.x (and before) indices. Remove two index dynamic disable test cases as the disability of index.mapper.dynamic is already removed for current version. Add a new test class for version test. --- .../index/mapper/MapperService.java | 13 +- .../mapper/DynamicMappingDisabledTests.java | 138 ------------------ .../index/mapper/DynamicMappingIT.java | 30 ---- .../mapper/DynamicMappingVersionTests.java | 80 ++++++++++ .../indices/settings/GetSettingsBlocksIT.java | 5 +- 5 files changed, 95 insertions(+), 171 deletions(-) delete mode 100644 core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java create mode 100644 core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 3a91d0e42ad73..c9851ed7a1d05 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -148,7 +148,18 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); this.mapperRegistry = mapperRegistry; - this.dynamic = this.indexSettings.getValue(INDEX_MAPPER_DYNAMIC_SETTING); + if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1)) { + if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings())) { + if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0_alpha1)) { + throw new IllegalArgumentException("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " was removed after version 6.0.0"); + } else { + DEPRECATION_LOGGER.deprecated("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " is deprecated since indices may not have more than one type anymore."); + } + } + this.dynamic = INDEX_MAPPER_DYNAMIC_DEFAULT; + } else { + this.dynamic = this.indexSettings.getValue(INDEX_MAPPER_DYNAMIC_SETTING); + } defaultMappingSource = "{\"_default_\":{}}"; if (logger.isTraceEnabled()) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java deleted file mode 100644 index 686bbafbcd23a..0000000000000 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.bulk.TransportBulkAction; -import org.elasticsearch.action.bulk.TransportShardBulkAction; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.AutoCreateIndex; -import org.elasticsearch.action.update.UpdateHelper; -import org.elasticsearch.client.Requests; -import org.elasticsearch.cluster.action.shard.ShardStateAction; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkService; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.MockTcpTransport; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportService; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -import java.util.Collections; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; -import static org.hamcrest.CoreMatchers.instanceOf; - -public class DynamicMappingDisabledTests extends ESSingleNodeTestCase { - - private static ThreadPool threadPool; - private ClusterService clusterService; - private TransportService transportService; - private TransportBulkAction transportBulkAction; - - @BeforeClass - public static void createThreadPool() { - threadPool = new TestThreadPool("DynamicMappingDisabledTests"); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - Settings settings = Settings.builder() - .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false) - .build(); - clusterService = createClusterService(threadPool); - Transport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE, - new NoneCircuitBreakerService(), new NamedWriteableRegistry(Collections.emptyList()), - new NetworkService(Collections.emptyList())); - transportService = new TransportService(clusterService.getSettings(), transport, threadPool, - TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null); - IndicesService indicesService = getInstanceFromNode(IndicesService.class); - ShardStateAction shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, threadPool); - ActionFilters actionFilters = new ActionFilters(Collections.emptySet()); - IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings); - AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, new ClusterSettings(settings, - ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), indexNameExpressionResolver); - UpdateHelper updateHelper = new UpdateHelper(settings, null); - TransportShardBulkAction shardBulkAction = new TransportShardBulkAction(settings, transportService, clusterService, - indicesService, threadPool, shardStateAction, null, updateHelper, actionFilters, indexNameExpressionResolver); - transportBulkAction = new TransportBulkAction(settings, threadPool, transportService, clusterService, - null, shardBulkAction, null, actionFilters, indexNameExpressionResolver, autoCreateIndex, System::currentTimeMillis); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); - clusterService.close(); - transportService.close(); - } - - - @AfterClass - public static void destroyThreadPool() { - ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); - // since static must set to null to be eligible for collection - threadPool = null; - } - - public void testDynamicDisabled() { - IndexRequest request = new IndexRequest("index", "type", "1"); - request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3); - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(request); - final AtomicBoolean gotResponse = new AtomicBoolean(); - - transportBulkAction.execute(bulkRequest, new ActionListener() { - @Override - public void onResponse(BulkResponse bulkResponse) { - BulkItemResponse itemResponse = bulkResponse.getItems()[0]; - assertTrue(itemResponse.isFailed()); - assertThat(itemResponse.getFailure().getCause(), instanceOf(IndexNotFoundException.class)); - assertEquals("no such index and [index.mapper.dynamic] is [false]", itemResponse.getFailure().getCause().getMessage()); - gotResponse.set(true); - } - - @Override - public void onFailure(Exception e) { - fail("unexpected failure in bulk action, expected failed bulk item"); - } - }); - - assertTrue(gotResponse.get()); - } -} diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index d183242ee19fe..4172a6172005e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -153,34 +153,4 @@ public void run() { assertTrue(client().prepareGet("index", "type", Integer.toString(i)).get().isExists()); } } - - public void testAutoCreateWithDisabledDynamicMappings() throws Exception { - assertAcked(client().admin().indices().preparePutTemplate("my_template") - .setCreate(true) - .setPatterns(Collections.singletonList("index_*")) - .addMapping("foo", "field", "type=keyword") - .setSettings(Settings.builder().put("index.mapper.dynamic", false).build()) - .get()); - - // succeeds since 'foo' has an explicit mapping in the template - indexRandom(true, false, client().prepareIndex("index_1", "foo", "1").setSource("field", "abc")); - - // fails since 'bar' does not have an explicit mapping in the template and dynamic template creation is disabled - TypeMissingException e1 = expectThrows(TypeMissingException.class, - () -> client().prepareIndex("index_2", "bar", "1").setSource("field", "abc").get()); - assertEquals("type[bar] missing", e1.getMessage()); - assertEquals("trying to auto create mapping, but dynamic mapping is disabled", e1.getCause().getMessage()); - - BulkResponse bulkResponse = client().prepareBulk().add(new IndexRequest("index_2", "bar", "2").source("field", "abc")).get(); - assertTrue(bulkResponse.hasFailures()); - BulkItemResponse.Failure firstFailure = bulkResponse.getItems()[0].getFailure(); - assertThat(firstFailure.getCause(), instanceOf(TypeMissingException.class)); - assertEquals("type[bar] missing", firstFailure.getCause().getMessage()); - assertEquals("trying to auto create mapping, but dynamic mapping is disabled", firstFailure.getCause().getCause().getMessage()); - - // make sure no mappings were created for bar - GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().addIndices("index_2").get(); - assertFalse(getIndexResponse.mappings().containsKey("bar")); - } - } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java new file mode 100644 index 0000000000000..94af6c5454493 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingVersionTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.TypeMissingException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import java.io.IOException; +import java.util.Collection; + +public class DynamicMappingVersionTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + public void testDynamicMappingDefault() throws IOException { + MapperService mapperService = createIndex("my-index").mapperService(); + DocumentMapper documentMapper = mapperService + .documentMapperWithAutoCreate("my-type").getDocumentMapper(); + + ParsedDocument parsedDoc = documentMapper.parse( + SourceToParse.source("my-index", "my-type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("foo", 3) + .endObject() + .bytes(), XContentType.JSON)); + + String expectedMapping = XContentFactory.jsonBuilder().startObject() + .startObject("my-type") + .startObject("properties") + .startObject("foo").field("type", "long") + .endObject().endObject().endObject().endObject().string(); + assertEquals(expectedMapping, parsedDoc.dynamicMappingsUpdate().toString()); + } + + public void testDynamicMappingSettingRemoval() { + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false) + .build(); + Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test-index", settings)); + assertEquals(e.getMessage(), "Setting index.mapper.dynamic was removed after version 6.0.0"); + } + + public void testDynamicMappingDisablePreEs6() { + Settings settingsPreEs6 = Settings.builder() + .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_0_0) + .build(); + MapperService preEs6MapperService = createIndex("pre-es6-index", settingsPreEs6).mapperService(); + Exception e = expectThrows(TypeMissingException.class, + () -> preEs6MapperService.documentMapperWithAutoCreate("pre-es6-type")); + assertEquals(e.getMessage(), "type[pre-es6-type] missing"); + } +} diff --git a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java index cb45a639c07eb..3d9b2aab7ad16 100644 --- a/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/indices/settings/GetSettingsBlocksIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESIntegTestCase; @@ -42,7 +43,7 @@ public void testGetSettingsWithBlocks() throws Exception { .setSettings(Settings.builder() .put("index.refresh_interval", -1) .put("index.merge.policy.expunge_deletes_allowed", "30") - .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false))); + .put(FieldMapper.IGNORE_MALFORMED_SETTING.getKey(), false))); for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_READ_ONLY_ALLOW_DELETE)) { try { @@ -51,7 +52,7 @@ public void testGetSettingsWithBlocks() throws Exception { assertThat(response.getIndexToSettings().size(), greaterThanOrEqualTo(1)); assertThat(response.getSetting("test", "index.refresh_interval"), equalTo("-1")); assertThat(response.getSetting("test", "index.merge.policy.expunge_deletes_allowed"), equalTo("30")); - assertThat(response.getSetting("test", MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey()), equalTo("false")); + assertThat(response.getSetting("test", FieldMapper.IGNORE_MALFORMED_SETTING.getKey()), equalTo("false")); } finally { disableIndexBlock("test", block); }