From b8f5abb6a3b8c74cfb41fe24a52c5f3e96c826c7 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 26 Jul 2018 09:02:58 +0200 Subject: [PATCH 01/87] Make sure _forcemerge respects `max_num_segments`. (#32291) An upcoming [Lucene change](https://issues.apache.org/jira/browse/LUCENE-7976) will make TieredMergePolicy respect the maximum merged segment size all the time, meaning it will possibly not respect the `max_num_segments` parameter anymore if the shard is larger than the maximum segment size. This change makes sure that `max_num_segments` is respected for now in order to give us time to think about how to integrate this change, and also to delay it until 7.0 as this might be a big-enough change for us to wait for a new major version. --- .../index/EsTieredMergePolicy.java | 120 ++++++++++++++++++ .../index/MergePolicyConfig.java | 2 +- .../index/EsTieredMergePolicyTests.java | 80 ++++++++++++ .../index/MergePolicySettingsTests.java | 43 +++---- 4 files changed, 222 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java create mode 100644 server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java diff --git a/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java new file mode 100644 index 0000000000000..27a8396190362 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/EsTieredMergePolicy.java @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.apache.lucene.index.FilterMergePolicy; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.TieredMergePolicy; + +import java.io.IOException; +import java.util.Map; + +/** + * Wrapper around {@link TieredMergePolicy} which doesn't respect + * {@link TieredMergePolicy#setMaxMergedSegmentMB(double)} on forced merges. + * See https://issues.apache.org/jira/browse/LUCENE-7976. + */ +final class EsTieredMergePolicy extends FilterMergePolicy { + + final TieredMergePolicy regularMergePolicy; + final TieredMergePolicy forcedMergePolicy; + + EsTieredMergePolicy() { + super(new TieredMergePolicy()); + regularMergePolicy = (TieredMergePolicy) in; + forcedMergePolicy = new TieredMergePolicy(); + forcedMergePolicy.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY); // unlimited + } + + @Override + public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, + Map segmentsToMerge, MergeContext mergeContext) throws IOException { + return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext); + } + + @Override + public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException { + return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext); + } + + public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) { + regularMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); + forcedMergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); + } + + public double getForceMergeDeletesPctAllowed() { + return forcedMergePolicy.getForceMergeDeletesPctAllowed(); + } + + public void setFloorSegmentMB(double mbFrac) { + regularMergePolicy.setFloorSegmentMB(mbFrac); + forcedMergePolicy.setFloorSegmentMB(mbFrac); + } + + public double getFloorSegmentMB() { + return regularMergePolicy.getFloorSegmentMB(); + } + + public void setMaxMergeAtOnce(int maxMergeAtOnce) { + regularMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + forcedMergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); + } + + public int getMaxMergeAtOnce() { + return regularMergePolicy.getMaxMergeAtOnce(); + } + + public void setMaxMergeAtOnceExplicit(int maxMergeAtOnceExplicit) { + regularMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); + forcedMergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); + } + + public int getMaxMergeAtOnceExplicit() { + return forcedMergePolicy.getMaxMergeAtOnceExplicit(); + } + + // only setter that must NOT delegate to the forced merge policy + public void setMaxMergedSegmentMB(double mbFrac) { + regularMergePolicy.setMaxMergedSegmentMB(mbFrac); + } + + public double getMaxMergedSegmentMB() { + return regularMergePolicy.getMaxMergedSegmentMB(); + } + + public void setSegmentsPerTier(double segmentsPerTier) { + regularMergePolicy.setSegmentsPerTier(segmentsPerTier); + forcedMergePolicy.setSegmentsPerTier(segmentsPerTier); + } + + public double getSegmentsPerTier() { + return regularMergePolicy.getSegmentsPerTier(); + } + + public void setReclaimDeletesWeight(double reclaimDeletesWeight) { + regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); + forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); + } + + public double getReclaimDeletesWeight() { + return regularMergePolicy.getReclaimDeletesWeight(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java index b3a0224f9834c..3131631586782 100644 --- a/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java +++ b/server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java @@ -116,7 +116,7 @@ */ public final class MergePolicyConfig { - private final TieredMergePolicy mergePolicy = new TieredMergePolicy(); + private final EsTieredMergePolicy mergePolicy = new EsTieredMergePolicy(); private final Logger logger; private final boolean mergesEnabled; diff --git a/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java new file mode 100644 index 0000000000000..fdee707d97a69 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/EsTieredMergePolicyTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.apache.lucene.index.TieredMergePolicy; +import org.elasticsearch.test.ESTestCase; + +public class EsTieredMergePolicyTests extends ESTestCase { + + public void testDefaults() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + assertEquals( + new TieredMergePolicy().getMaxMergedSegmentMB(), + policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); + // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double + assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); + } + + public void testSetMaxMergedSegmentMB() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergedSegmentMB(10 * 1024); + assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d); + // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double + assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d); + } + + public void testSetForceMergeDeletesPctAllowed() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setForceMergeDeletesPctAllowed(42); + assertEquals(42, policy.forcedMergePolicy.getForceMergeDeletesPctAllowed(), 0); + } + + public void testSetFloorSegmentMB() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setFloorSegmentMB(42); + assertEquals(42, policy.regularMergePolicy.getFloorSegmentMB(), 0); + assertEquals(42, policy.forcedMergePolicy.getFloorSegmentMB(), 0); + } + + public void testSetMaxMergeAtOnce() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergeAtOnce(42); + assertEquals(42, policy.regularMergePolicy.getMaxMergeAtOnce()); + } + + public void testSetMaxMergeAtOnceExplicit() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setMaxMergeAtOnceExplicit(42); + assertEquals(42, policy.forcedMergePolicy.getMaxMergeAtOnceExplicit()); + } + + public void testSetSegmentsPerTier() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setSegmentsPerTier(42); + assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0); + } + + public void testSetReclaimDeletesWeight() { + EsTieredMergePolicy policy = new EsTieredMergePolicy(); + policy.setReclaimDeletesWeight(42); + assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java index ff60a206229d6..d8b03ffeedbfc 100644 --- a/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/MergePolicySettingsTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.TieredMergePolicy; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -76,43 +75,43 @@ public void testUpdateSettings() throws IOException { public void testTieredMergePolicySettingsUpdate() throws IOException { IndexSettings indexSettings = indexSettings(Settings.EMPTY); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING.getKey(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED + 1.0d, 0.0d); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMbFrac(), 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB)).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb() + 1, ByteSizeUnit.MB).getMbFrac(), 0.001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE - 1); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING.getKey(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT - 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT-1); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getMbFrac(), 0.0001); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build())); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0); indexSettings.updateIndexMetaData(newIndexMeta("index", EMPTY_SETTINGS)); // see if defaults are restored - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); - assertEquals(((TieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getForceMergeDeletesPctAllowed(), MergePolicyConfig.DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getFloorSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_FLOOR_SEGMENT.getMb(), ByteSizeUnit.MB).getMbFrac(), 0.00); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0); + assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0); } public Settings build(String value) { From ed44b1bb4eaf52fbcfeebdc7d701508caa149db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 26 Jul 2018 11:15:53 +0200 Subject: [PATCH 02/87] Rank-Eval: Reduce scope of an unchecked supression We should only supress the unchecked warnings on ConstructingObjectParser. --- .../java/org/elasticsearch/index/rankeval/RatedRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 79dd693b3ac3c..2fe3f1922d509 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -70,7 +70,6 @@ * } * */ -@SuppressWarnings("unchecked") public class RatedRequest implements Writeable, ToXContentObject { private final String id; private final List summaryFields; @@ -250,6 +249,7 @@ public void addSummaryFields(List summaryFields) { private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id"); + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("request", a -> new RatedRequest((String) a[0], (List) a[1], (SearchSourceBuilder) a[2], (Map) a[3], (String) a[4])); From ac0125fe2c088fda6ac661725c02f61ba1628e82 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 26 Jul 2018 08:05:49 -0400 Subject: [PATCH 03/87] Introduce index store plugins (#32375) Today we allow plugins to add index store implementations yet we are not doing this in our new way of managing plugins as pull versus push. That is, today we still allow plugins to push index store providers via an on index module call where they can turn around and add an index store. Aside from being inconsistent with how we manage plugins today where we would look to pull such implementations from plugins at node creation time, it also means that we do not know at a top-level (for example, in the indices service) which index stores are available. This commit addresses this by adding a dedicated plugin type for index store plugins, removing the index module hook for adding index stores, and by aggregating these into the top-level of the indices service. --- .../plugin/store/smb/SMBStorePlugin.java | 20 ++++-- .../org/elasticsearch/index/IndexModule.java | 40 ++++------- .../elasticsearch/indices/IndicesService.java | 10 ++- .../java/org/elasticsearch/node/Node.java | 12 +++- .../plugins/IndexStorePlugin.java | 42 +++++++++++ .../elasticsearch/index/IndexModuleTests.java | 61 ++++++++-------- .../plugins/IndexStorePluginTests.java | 72 +++++++++++++++++++ .../test/store/MockFSIndexStore.java | 10 ++- .../xpack/watcher/WatcherPluginTests.java | 3 +- 9 files changed, 201 insertions(+), 69 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/plugins/IndexStorePlugin.java create mode 100644 server/src/test/java/org/elasticsearch/plugins/IndexStorePluginTests.java diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java index 241f3d77e8b87..a2b0f46345231 100644 --- a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java +++ b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java @@ -19,16 +19,26 @@ package org.elasticsearch.plugin.store.smb; -import org.elasticsearch.index.IndexModule; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.smbmmapfs.SmbMmapFsIndexStore; import org.elasticsearch.index.store.smbsimplefs.SmbSimpleFsIndexStore; +import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.plugins.Plugin; -public class SMBStorePlugin extends Plugin { +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +public class SMBStorePlugin extends Plugin implements IndexStorePlugin { @Override - public void onIndexModule(IndexModule indexModule) { - indexModule.addIndexStore("smb_mmap_fs", SmbMmapFsIndexStore::new); - indexModule.addIndexStore("smb_simple_fs", SmbSimpleFsIndexStore::new); + public Map> getIndexStoreFactories() { + final Map> indexStoreFactories = new HashMap<>(2); + indexStoreFactories.put("smb_mmap_fs", SmbMmapFsIndexStore::new); + indexStoreFactories.put("smb_simple_fs", SmbSimpleFsIndexStore::new); + return Collections.unmodifiableMap(indexStoreFactories); } + } diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 9e859a16956c8..715b78b14ffdb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -49,6 +49,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; @@ -74,7 +75,7 @@ * {@link #addSimilarity(String, TriFunction)} while existing Providers can be referenced through Settings under the * {@link IndexModule#SIMILARITY_SETTINGS_PREFIX} prefix along with the "type" value. For example, to reference the * {@link BM25Similarity}, the configuration {@code "index.similarity.my_similarity.type : "BM25"} can be used. - *
  • {@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link #addIndexStore(String, Function)}
    • + *
  • {@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link IndexStorePlugin}
    • *
  • {@link IndexEventListener} - Custom {@link IndexEventListener} instances can be registered via * {@link #addIndexEventListener(IndexEventListener)}
    • *
  • Settings update listener - Custom settings update listener can be registered via @@ -109,7 +110,7 @@ public final class IndexModule { private SetOnce indexSearcherWrapper = new SetOnce<>(); private final Set indexEventListeners = new HashSet<>(); private final Map> similarities = new HashMap<>(); - private final Map> storeTypes = new HashMap<>(); + private final Map> indexStoreFactories; private final SetOnce> forceQueryCacheProvider = new SetOnce<>(); private final List searchOperationListeners = new ArrayList<>(); private final List indexOperationListeners = new ArrayList<>(); @@ -119,16 +120,22 @@ public final class IndexModule { * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins * via {@link org.elasticsearch.plugins.PluginsService#onIndexModule(IndexModule)}. * - * @param indexSettings the index settings - * @param analysisRegistry the analysis registry - * @param engineFactory the engine factory + * @param indexSettings the index settings + * @param analysisRegistry the analysis registry + * @param engineFactory the engine factory + * @param indexStoreFactories the available store types */ - public IndexModule(final IndexSettings indexSettings, final AnalysisRegistry analysisRegistry, final EngineFactory engineFactory) { + public IndexModule( + final IndexSettings indexSettings, + final AnalysisRegistry analysisRegistry, + final EngineFactory engineFactory, + final Map> indexStoreFactories) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; this.engineFactory = Objects.requireNonNull(engineFactory); this.searchOperationListeners.add(new SearchSlowLog(indexSettings)); this.indexOperationListeners.add(new IndexingSlowLog(indexSettings)); + this.indexStoreFactories = Collections.unmodifiableMap(indexStoreFactories); } /** @@ -245,25 +252,6 @@ public void addIndexOperationListener(IndexingOperationListener listener) { this.indexOperationListeners.add(listener); } - /** - * Adds an {@link IndexStore} type to this index module. Typically stores are registered with a reference to - * it's constructor: - * 
    -     *     indexModule.addIndexStore("my_store_type", MyStore::new);
-     *
    - * - * @param type the type to register - * @param provider the instance provider / factory method - */ - public void addIndexStore(String type, Function provider) { - ensureNotFrozen(); - if (storeTypes.containsKey(type)) { - throw new IllegalArgumentException("key [" + type +"] already registered"); - } - storeTypes.put(type, provider); - } - - /** * Registers the given {@link Similarity} with the given name. * The function takes as parameters: