From 8c326b42a1602108efb45ac4d9ca6bc356ce15ac Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 24 May 2018 22:33:00 -0400 Subject: [PATCH 1/6] Do not return metadata customs by default Returning metadata customs by default is dangerous because they could be from a plugin that is not installed on a transport client. This commit modifies the cluster state action so that metadata customs are not returned instead of explicitly asked for. We make the same change on the REST layer, and we require that if metadata customs are requested, so is metadata. --- .../cluster/state/ClusterStateRequest.java | 27 ++- .../state/ClusterStateRequestBuilder.java | 5 + .../state/TransportClusterStateAction.java | 15 +- .../elasticsearch/cluster/ClusterState.java | 2 +- .../admin/cluster/RestClusterStateAction.java | 1 + .../admin/cluster/state/ClusterStateIT.java | 183 ++++++++++++++++++ .../DedicatedClusterSnapshotRestoreIT.java | 11 +- .../snapshots/RepositoriesIT.java | 6 +- 8 files changed, 238 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index 33a20332526bf..aa34c18d86264 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.state; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -29,11 +30,14 @@ import java.io.IOException; +import static org.elasticsearch.action.ValidateActions.addValidationError; + public class ClusterStateRequest extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private boolean routingTable = true; private boolean nodes = true; private boolean metaData = true; + private boolean metaDataCustoms = false; private boolean blocks = true; private boolean customs = true; private String[] indices = Strings.EMPTY_ARRAY; @@ -47,6 +51,9 @@ public ClusterStateRequest(StreamInput in) throws IOException { routingTable = in.readBoolean(); nodes = in.readBoolean(); metaData = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_6_3_0)) { + metaDataCustoms = in.readBoolean(); + } blocks = in.readBoolean(); customs = in.readBoolean(); indices = in.readStringArray(); @@ -59,6 +66,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(routingTable); out.writeBoolean(nodes); out.writeBoolean(metaData); + if (out.getVersion().onOrAfter(Version.V_6_3_0)) { + out.writeBoolean(metaDataCustoms); + } out.writeBoolean(blocks); out.writeBoolean(customs); out.writeStringArray(indices); @@ -67,13 +77,18 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - return null; + if (metaData == false && metaDataCustoms) { + return addValidationError("metadata customs were requested without requesting metadata", null); + } else { + return null; + } } public ClusterStateRequest all() { routingTable = true; nodes = true; metaData = true; + metaDataCustoms = true; blocks = true; customs = true; indices = Strings.EMPTY_ARRAY; @@ -84,6 +99,7 @@ public ClusterStateRequest clear() { routingTable = false; nodes = false; metaData = false; + metaDataCustoms = false; blocks = false; customs = false; indices = Strings.EMPTY_ARRAY; @@ -117,6 +133,15 @@ public ClusterStateRequest metaData(boolean metaData) { return this; } + public boolean metaDataCustoms() { + return metaDataCustoms; + } + + public ClusterStateRequest metaDataCustoms(boolean metaDataCustoms) { + this.metaDataCustoms = metaDataCustoms; + return this; + } + public boolean blocks() { return blocks; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java index 524e167e3a265..9d1aef22564ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java @@ -59,6 +59,11 @@ public ClusterStateRequestBuilder setMetaData(boolean filter) { return this; } + public ClusterStateRequestBuilder setMetaDataCustoms(boolean filter) { + request.metaDataCustoms(filter); + return this; + } + /** * Should the cluster state result include the {@link org.elasticsearch.cluster.node.DiscoveryNodes}. Defaults * to {@code true}. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index b7ef075a59afa..c7faf8fb5e309 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.function.Predicate; import static org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState; @@ -115,13 +116,19 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS mdBuilder = MetaData.builder(currentState.metaData()); } - // filter out metadata that shouldn't be returned by the API - for (ObjectObjectCursor custom : currentState.metaData().customs()) { - if (custom.value.context().contains(MetaData.XContentContext.API) == false) { - mdBuilder.removeCustom(custom.key); + final Predicate predicate; + if (request.metaDataCustoms()) { + predicate = c -> c.context().contains(MetaData.XContentContext.API); + } else { + predicate = c -> false; + } + for (final ObjectObjectCursor cursor : currentState.metaData().customs()) { + if (predicate.test(cursor.value) == false) { + mdBuilder.removeCustom(cursor.key); } } } + builder.metaData(mdBuilder); if (request.customs()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 2b991d1dc611a..b79fe2949b06b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -22,7 +22,6 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -277,6 +276,7 @@ public enum Metric { BLOCKS("blocks"), NODES("nodes"), METADATA("metadata"), + METADATA_CUSTOMS("metadata_customs"), ROUTING_TABLE("routing_table"), ROUTING_NODES("routing_nodes"), CUSTOMS("customs"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java index 6e55ef3671ba0..e2f0a18cecb2d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java @@ -85,6 +85,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC clusterStateRequest.routingTable( metrics.contains(ClusterState.Metric.ROUTING_TABLE) || metrics.contains(ClusterState.Metric.ROUTING_NODES)); clusterStateRequest.metaData(metrics.contains(ClusterState.Metric.METADATA)); + clusterStateRequest.metaDataCustoms(metrics.contains(ClusterState.Metric.METADATA_CUSTOMS)); clusterStateRequest.blocks(metrics.contains(ClusterState.Metric.BLOCKS)); clusterStateRequest.customs(metrics.contains(ClusterState.Metric.CUSTOMS)); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java new file mode 100644 index 0000000000000..f9d4ae7dbfbfb --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.state; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasToString; + +public class ClusterStateIT extends ESSingleNodeTestCase { + + public static class CustomPlugin extends Plugin { + + public CustomPlugin() { + + } + + static class CustomPluginCustom implements MetaData.Custom { + + @Override + public EnumSet context() { + return MetaData.ALL_CONTEXTS; + } + + @Override + public Diff diff(final MetaData.Custom previousState) { + return null; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + { + + } + builder.endObject(); + return builder; + } + } + + @Override + public List getNamedWriteables() { + return super.getNamedWriteables(); + } + + public static final String TYPE = "custom_plugin"; + + private final AtomicBoolean installed = new AtomicBoolean(); + + @Override + public Collection createComponents( + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final ResourceWatcherService resourceWatcherService, + final ScriptService scriptService, + final NamedXContentRegistry xContentRegistry, + final Environment environment, + final NodeEnvironment nodeEnvironment, + final NamedWriteableRegistry namedWriteableRegistry) { + clusterService.addListener(event -> { + final ClusterState state = event.state(); + if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { + return; + } + + final MetaData metaData = state.metaData(); + if (state.nodes().isLocalNodeElectedMaster()) { + if (metaData.custom(CustomPlugin.TYPE) == null) { + if (installed.compareAndSet(false, true)) { + clusterService.submitStateUpdateTask("install-metadata-custom", new ClusterStateUpdateTask(Priority.URGENT) { + + @Override + public ClusterState execute(ClusterState currentState) { + if (currentState.custom(CustomPlugin.TYPE) == null) { + final MetaData.Builder builder = MetaData.builder(currentState.metaData()); + builder.putCustom(CustomPlugin.TYPE, new CustomPluginCustom()); + return ClusterState.builder(currentState).metaData(builder).build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + throw new AssertionError(e); + } + + }); + } + } + } + + }); + return Collections.emptyList(); + } + } + + @Override + protected Collection> getPlugins() { + return Stream.concat(super.getPlugins().stream(), Stream.of(CustomPlugin.class)).collect(Collectors.toCollection(ArrayList::new)); + } + + public void testRequestCustoms() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(true).get(); + assertTrue(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + } + + public void testDoNotRequestCustoms() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(false).get(); + assertFalse(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + } + + public void testRequestCustomsDefault() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).get(); + assertFalse(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + } + + public void testValidation() { + final ClusterStateRequest request = new ClusterStateRequest(); + request.metaData(false); + request.metaDataCustoms(true); + final ActionRequestValidationException e = request.validate(); + assertThat(e, hasToString(containsString("metadata customs were requested without requesting metadata"))); + } + +} \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 1dc853db59467..5061db1a33ca7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -94,7 +94,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -303,7 +305,7 @@ public void testRestoreCustomMetadata() throws Exception { assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); logger.info("--> check that custom persistent metadata was restored"); - ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + ClusterState clusterState = client.admin().cluster().prepareState().setMetaDataCustoms(true).get().getState(); logger.info("Cluster state: {}", clusterState); MetaData metaData = clusterState.getMetaData(); assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); @@ -316,7 +318,7 @@ public void testRestoreCustomMetadata() throws Exception { ensureYellow(); logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); - clusterState = client().admin().cluster().prepareState().get().getState(); + clusterState = client().admin().cluster().prepareState().setMetaDataCustoms(true).get().getState(); logger.info("Cluster state: {}", clusterState); metaData = clusterState.getMetaData(); assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); @@ -729,7 +731,10 @@ public void sendResponse(RestResponse response) { RestClusterStateAction clusterStateAction = new RestClusterStateAction(nodeSettings, mock(RestController.class), internalCluster().getInstance(SettingsFilter.class)); - RestRequest clusterStateRequest = new FakeRestRequest(); + RestRequest clusterStateRequest = + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withParams(new HashMap<>(Collections.singletonMap("metric", "metadata,metadata_customs"))) + .build(); final CountDownLatch clusterStateLatch = new CountDownLatch(1); final AtomicReference clusterStateError = new AtomicReference<>(); clusterStateAction.handleRequest(clusterStateRequest, new AbstractRestChannel(clusterStateRequest, true) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 23cb579bfdc92..7cbfa5b58d614 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -40,7 +40,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -67,7 +66,8 @@ public void testRepositoryCreation() throws Exception { assertThat(FileSystemUtils.files(location).length, equalTo(numberOfFiles)); logger.info("--> check that repository is really there"); - ClusterStateResponse clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).get(); + ClusterStateResponse clusterStateResponse = + client.admin().cluster().prepareState().clear().setMetaData(true).setMetaDataCustoms(true).get(); MetaData metaData = clusterStateResponse.getState().getMetaData(); RepositoriesMetaData repositoriesMetaData = metaData.custom(RepositoriesMetaData.TYPE); assertThat(repositoriesMetaData, notNullValue()); @@ -82,7 +82,7 @@ public void testRepositoryCreation() throws Exception { assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); logger.info("--> check that both repositories are in cluster state"); - clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).get(); + clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).setMetaDataCustoms(true).get(); metaData = clusterStateResponse.getState().getMetaData(); repositoriesMetaData = metaData.custom(RepositoriesMetaData.TYPE); assertThat(repositoriesMetaData, notNullValue()); From 791d9f768eb1c77781fee7a9c44f1649750b9aed Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 24 May 2018 22:40:16 -0400 Subject: [PATCH 2/6] Add missing newline --- .../action/admin/cluster/state/ClusterStateIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java index f9d4ae7dbfbfb..dfbb19689095c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java @@ -180,4 +180,4 @@ public void testValidation() { assertThat(e, hasToString(containsString("metadata customs were requested without requesting metadata"))); } -} \ No newline at end of file +} From 76b297c55c56007b955cc3b1293c5a23a9ce55f7 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 25 May 2018 07:34:07 -0400 Subject: [PATCH 3/6] Start with 7.0.0, fix on backport --- .../action/admin/cluster/state/ClusterStateRequest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index aa34c18d86264..0b1798650dc91 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -51,7 +51,7 @@ public ClusterStateRequest(StreamInput in) throws IOException { routingTable = in.readBoolean(); nodes = in.readBoolean(); metaData = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_6_3_0)) { + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { metaDataCustoms = in.readBoolean(); } blocks = in.readBoolean(); @@ -66,7 +66,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(routingTable); out.writeBoolean(nodes); out.writeBoolean(metaData); - if (out.getVersion().onOrAfter(Version.V_6_3_0)) { + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { out.writeBoolean(metaDataCustoms); } out.writeBoolean(blocks); From e34b58a5bb0d9b8dfd1409338de797bdea17b880 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 25 May 2018 07:54:01 -0400 Subject: [PATCH 4/6] Adjust tests --- .../admin/cluster/state/ClusterStateIT.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java index dfbb19689095c..aef3b90c1ca22 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -41,17 +42,22 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.not; public class ClusterStateIT extends ESSingleNodeTestCase { @@ -159,23 +165,27 @@ protected Collection> getPlugins() { public void testRequestCustoms() { final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(true).get(); - assertTrue(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, hasItem(CustomPlugin.TYPE)); } public void testDoNotRequestCustoms() { final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(false).get(); - assertFalse(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, not(hasItem(CustomPlugin.TYPE))); } public void testRequestCustomsDefault() { final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).get(); - assertFalse(state.getState().metaData().customs().containsKey(CustomPlugin.TYPE)); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, not(hasItem(CustomPlugin.TYPE))); } public void testValidation() { - final ClusterStateRequest request = new ClusterStateRequest(); - request.metaData(false); - request.metaDataCustoms(true); + final ClusterStateRequest request = new ClusterStateRequest().metaData(false).metaDataCustoms(true); final ActionRequestValidationException e = request.validate(); assertThat(e, hasToString(containsString("metadata customs were requested without requesting metadata"))); } From afa50d26eb139527769c740147dd049c633a9742 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 27 May 2018 16:02:31 -0400 Subject: [PATCH 5/6] Fix clean after ML tests --- .../core/ml/integration/MlRestTestStateCleaner.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java index 8e326e3556b59..7af178b588526 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.integration; import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.rest.ESRestTestCase; @@ -35,8 +36,10 @@ public void clearMlMetadata() throws IOException { @SuppressWarnings("unchecked") private void deleteAllDatafeeds() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.datafeeds"))); + final Request datafeedsRequest = new Request("GET", "/_cluster/state/metadata,metadata_customs"); + datafeedsRequest.addParameter("filter_path", "metadata.ml.datafeeds"); + final Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest(datafeedsRequest)); + List> datafeeds = (List>) XContentMapValues.extractValue("metadata.ml.datafeeds", clusterStateAsMap); if (datafeeds == null) { @@ -75,8 +78,9 @@ private void deleteAllDatafeeds() throws IOException { } private void deleteAllJobs() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.jobs"))); + final Request jobsRequest = new Request("GET", "/_cluster/state/metadata,metadata_customs"); + jobsRequest.addParameter("filter_path", "metadata.ml.jobs"); + final Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest(jobsRequest)); @SuppressWarnings("unchecked") List> jobConfigs = (List>) XContentMapValues.extractValue("metadata.ml.jobs", clusterStateAsMap); From 1a9a7242107a79730566600881b7c3d5b2754b85 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 28 May 2018 10:38:45 -0400 Subject: [PATCH 6/6] Fix more ML tests --- .../test/resources/rest-api-spec/test/ml/jobs_crud.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 9ed14c2f860ef..a3dc6173d5c2f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -549,7 +549,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} @@ -562,7 +562,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: metadata.persistent_tasks.tasks: [] @@ -790,7 +790,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} @@ -804,7 +804,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: metadata.persistent_tasks.tasks: []