From b2afce865be03153d411c5c9c34a8dc22c59d88a Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 11 Sep 2017 12:40:41 +0200 Subject: [PATCH 1/4] Simplify GatewayMetaState deps --- .../java/org/elasticsearch/gateway/GatewayMetaState.java | 2 -- .../gateway/TransportNodesListGatewayMetaState.java | 9 +++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 9d57392030ce6..898a02e2deb9f 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -71,13 +71,11 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA @Inject public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService, - TransportNodesListGatewayMetaState nodesListGatewayMetaState, MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws Exception { super(settings); this.nodeEnv = nodeEnv; this.metaStateService = metaStateService; - nodesListGatewayMetaState.init(this); if (DiscoveryNode.isDataNode(settings)) { ensureNoPre019ShardState(nodeEnv); diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java index 4247ec131a33d..04253dfabb18a 100644 --- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java @@ -52,19 +52,16 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction list(String[] nodesIds, @Nullable TimeValue timeout) { From 265056f7c2c2a1818c29ad41d193d860d8723a57 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 18 Sep 2017 12:10:03 +0200 Subject: [PATCH 2/4] deguicify metastate --- .../metadata/MetaDataIndexUpgradeService.java | 4 ---- .../org/elasticsearch/gateway/GatewayMetaState.java | 13 +++++-------- .../org/elasticsearch/gateway/GatewayModule.java | 1 - .../org/elasticsearch/gateway/MetaStateService.java | 2 +- core/src/main/java/org/elasticsearch/node/Node.java | 9 +++++++-- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 269657367dcfa..2ff5fd5c2b217 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -18,13 +18,11 @@ */ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.Version; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -35,7 +33,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.plugins.Plugin; import java.util.AbstractMap; import java.util.Collection; @@ -59,7 +56,6 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { private final IndexScopedSettings indexScopedSettings; private final UnaryOperator upgraders; - @Inject public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings, Collection> indexMetaDataUpgraders) { diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 898a02e2deb9f..f205708c03d63 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.IndexFolderUpgrader; @@ -69,10 +68,8 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA private volatile Set previouslyWrittenIndices = emptySet(); - @Inject public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateService metaStateService, - MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) - throws Exception { + MetaDataIndexUpgradeService metaDataIndexUpgradeService, MetaDataUpgrader metaDataUpgrader) throws IOException { super(settings); this.nodeEnv = nodeEnv; this.metaStateService = metaStateService; @@ -106,7 +103,7 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer long startNS = System.nanoTime(); metaStateService.loadFullState(); logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); - } catch (Exception e) { + } catch (IOException e) { logger.error("failed to read local state, exiting...", e); throw e; } @@ -208,7 +205,7 @@ protected static boolean isDataOnlyNode(ClusterState state) { /** * Throws an IAE if a pre 0.19 state is detected */ - private void ensureNoPre019State() throws Exception { + private void ensureNoPre019State() throws IOException { for (Path dataLocation : nodeEnv.nodeDataPaths()) { final Path stateLocation = dataLocation.resolve(MetaDataStateFormat.STATE_DIR_NAME); if (!Files.exists(stateLocation)) { @@ -240,7 +237,7 @@ private void ensureNoPre019State() throws Exception { */ static MetaData upgradeMetaData(MetaData metaData, MetaDataIndexUpgradeService metaDataIndexUpgradeService, - MetaDataUpgrader metaDataUpgrader) throws Exception { + MetaDataUpgrader metaDataUpgrader) throws IOException { // upgrade index meta data boolean changed = false; final MetaData.Builder upgradedMetaData = MetaData.builder(metaData); @@ -286,7 +283,7 @@ private static boolean applyPluginUpgraders(ImmutableOpenMap Collection> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream() .map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList()); final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders); + final MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, + indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders); + final GatewayMetaState gatewayMetaState = new GatewayMetaState(settings, nodeEnvironment, metaStateService, + metaDataIndexUpgradeService, metaDataUpgrader); new TemplateUpgradeService(settings, client, clusterService, threadPool, indexTemplateMetaDataUpgraders); final Transport transport = networkModule.getTransportSupplier().get(); final TransportService transportService = newTransportService(settings, transport, threadPool, @@ -473,9 +478,9 @@ protected Node(final Environment environment, Collection b.bind(TransportService.class).toInstance(transportService); b.bind(NetworkService.class).toInstance(networkService); b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService())); - b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry, - indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders)); + b.bind(MetaDataIndexUpgradeService.class).toInstance(metaDataIndexUpgradeService); b.bind(ClusterInfoService.class).toInstance(clusterInfoService); + b.bind(GatewayMetaState.class).toInstance(gatewayMetaState); b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery()); { RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); From debce3cd1f2e74a86543b96722df160556fb8ba1 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 12 Sep 2017 07:43:13 +0200 Subject: [PATCH 3/4] Gateway is the master-level component --- .../java/org/elasticsearch/gateway/Gateway.java | 17 ++--------------- .../elasticsearch/gateway/GatewayService.java | 4 +++- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java index 2e258ca54de69..f4d191ac28a8a 100644 --- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java +++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java @@ -23,9 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; @@ -39,27 +37,23 @@ import java.util.Arrays; import java.util.Map; -public class Gateway extends AbstractComponent implements ClusterStateApplier { +public class Gateway extends AbstractComponent { private final ClusterService clusterService; - private final GatewayMetaState metaState; - private final TransportNodesListGatewayMetaState listGatewayMetaState; private final int minimumMasterNodes; private final IndicesService indicesService; - public Gateway(Settings settings, ClusterService clusterService, GatewayMetaState metaState, + public Gateway(Settings settings, ClusterService clusterService, TransportNodesListGatewayMetaState listGatewayMetaState, IndicesService indicesService) { super(settings); this.indicesService = indicesService; this.clusterService = clusterService; - this.metaState = metaState; this.listGatewayMetaState = listGatewayMetaState; this.minimumMasterNodes = ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings); - clusterService.addLowPriorityApplier(this); } public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException { @@ -174,13 +168,6 @@ private void logInvalidSetting(String settingType, Map.Entry e, ex); } - @Override - public void applyClusterState(final ClusterChangedEvent event) { - // order is important, first metaState, and then shardsState - // so dangling indices will be recorded - metaState.applyClusterState(event); - } - public interface GatewayStateRecoveredListener { void onSuccess(ClusterState build); diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java index 6b61e03443eae..91ce90bd8b58c 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java @@ -95,7 +95,7 @@ public GatewayService(Settings settings, AllocationService allocationService, Cl TransportNodesListGatewayMetaState listGatewayMetaState, IndicesService indicesService) { super(settings); - this.gateway = new Gateway(settings, clusterService, metaState, listGatewayMetaState, + this.gateway = new Gateway(settings, clusterService, listGatewayMetaState, indicesService); this.allocationService = allocationService; this.clusterService = clusterService; @@ -121,6 +121,8 @@ public GatewayService(Settings settings, AllocationService allocationService, Cl // TODO: change me once the minimum_master_nodes is changed too recoverAfterMasterNodes = settings.getAsInt("discovery.zen.minimum_master_nodes", -1); } + + clusterService.addLowPriorityApplier(metaState); } @Override From 7162170331bbec76f9ed2554d53c37a4aa2d620f Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Wed, 20 Sep 2017 10:10:18 +0200 Subject: [PATCH 4/4] fix test Previously this exception would be thrown as part of a big Guice creation error. Guice exposes the messages of all sub-exceptions in one big chunk. We now have to be more selective. --- .../main/java/org/elasticsearch/gateway/GatewayMetaState.java | 2 +- .../bwcompat/RecoveryWithUnsupportedIndicesIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index f205708c03d63..719626b7e1870 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -103,7 +103,7 @@ public GatewayMetaState(Settings settings, NodeEnvironment nodeEnv, MetaStateSer long startNS = System.nanoTime(); metaStateService.loadFullState(); logger.debug("took {} to load state", TimeValue.timeValueMillis(TimeValue.nsecToMSec(System.nanoTime() - startNS))); - } catch (IOException e) { + } catch (Exception e) { logger.error("failed to read local state, exiting...", e); throw e; } diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java index 50f328db39306..51ff79a4a2fa2 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java @@ -91,7 +91,7 @@ public void testUpgradeStartClusterOn_0_20_6() throws Exception { internalCluster().startNode(nodeSettings); fail(); } catch (Exception ex) { - assertThat(ex.getMessage(), containsString(" was created before v2.0.0.beta1 and wasn't upgraded")); + assertThat(ex.getCause().getCause().getMessage(), containsString(" was created before v2.0.0.beta1 and wasn't upgraded")); } } }