diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java index 33a20332526bf..0b1798650dc91 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.cluster.state; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -29,11 +30,14 @@ import java.io.IOException; +import static org.elasticsearch.action.ValidateActions.addValidationError; + public class ClusterStateRequest extends MasterNodeReadRequest implements IndicesRequest.Replaceable { private boolean routingTable = true; private boolean nodes = true; private boolean metaData = true; + private boolean metaDataCustoms = false; private boolean blocks = true; private boolean customs = true; private String[] indices = Strings.EMPTY_ARRAY; @@ -47,6 +51,9 @@ public ClusterStateRequest(StreamInput in) throws IOException { routingTable = in.readBoolean(); nodes = in.readBoolean(); metaData = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + metaDataCustoms = in.readBoolean(); + } blocks = in.readBoolean(); customs = in.readBoolean(); indices = in.readStringArray(); @@ -59,6 +66,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(routingTable); out.writeBoolean(nodes); out.writeBoolean(metaData); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeBoolean(metaDataCustoms); + } out.writeBoolean(blocks); out.writeBoolean(customs); out.writeStringArray(indices); @@ -67,13 +77,18 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - return null; + if (metaData == false && metaDataCustoms) { + return addValidationError("metadata customs were requested without requesting metadata", null); + } else { + return null; + } } public ClusterStateRequest all() { routingTable = true; nodes = true; metaData = true; + metaDataCustoms = true; blocks = true; customs = true; indices = Strings.EMPTY_ARRAY; @@ -84,6 +99,7 @@ public ClusterStateRequest clear() { routingTable = false; nodes = false; metaData = false; + metaDataCustoms = false; blocks = false; customs = false; indices = Strings.EMPTY_ARRAY; @@ -117,6 +133,15 @@ public ClusterStateRequest metaData(boolean metaData) { return this; } + public boolean metaDataCustoms() { + return metaDataCustoms; + } + + public ClusterStateRequest metaDataCustoms(boolean metaDataCustoms) { + this.metaDataCustoms = metaDataCustoms; + return this; + } + public boolean blocks() { return blocks; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java index 524e167e3a265..9d1aef22564ff 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java @@ -59,6 +59,11 @@ public ClusterStateRequestBuilder setMetaData(boolean filter) { return this; } + public ClusterStateRequestBuilder setMetaDataCustoms(boolean filter) { + request.metaDataCustoms(filter); + return this; + } + /** * Should the cluster state result include the {@link org.elasticsearch.cluster.node.DiscoveryNodes}. Defaults * to {@code true}. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index b7ef075a59afa..c7faf8fb5e309 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.function.Predicate; import static org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState; @@ -115,13 +116,19 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS mdBuilder = MetaData.builder(currentState.metaData()); } - // filter out metadata that shouldn't be returned by the API - for (ObjectObjectCursor custom : currentState.metaData().customs()) { - if (custom.value.context().contains(MetaData.XContentContext.API) == false) { - mdBuilder.removeCustom(custom.key); + final Predicate predicate; + if (request.metaDataCustoms()) { + predicate = c -> c.context().contains(MetaData.XContentContext.API); + } else { + predicate = c -> false; + } + for (final ObjectObjectCursor cursor : currentState.metaData().customs()) { + if (predicate.test(cursor.value) == false) { + mdBuilder.removeCustom(cursor.key); } } } + builder.metaData(mdBuilder); if (request.customs()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 2b991d1dc611a..b79fe2949b06b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -22,7 +22,6 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -277,6 +276,7 @@ public enum Metric { BLOCKS("blocks"), NODES("nodes"), METADATA("metadata"), + METADATA_CUSTOMS("metadata_customs"), ROUTING_TABLE("routing_table"), ROUTING_NODES("routing_nodes"), CUSTOMS("customs"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java index 6e55ef3671ba0..e2f0a18cecb2d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java @@ -85,6 +85,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC clusterStateRequest.routingTable( metrics.contains(ClusterState.Metric.ROUTING_TABLE) || metrics.contains(ClusterState.Metric.ROUTING_NODES)); clusterStateRequest.metaData(metrics.contains(ClusterState.Metric.METADATA)); + clusterStateRequest.metaDataCustoms(metrics.contains(ClusterState.Metric.METADATA_CUSTOMS)); clusterStateRequest.blocks(metrics.contains(ClusterState.Metric.BLOCKS)); clusterStateRequest.customs(metrics.contains(ClusterState.Metric.CUSTOMS)); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java new file mode 100644 index 0000000000000..aef3b90c1ca22 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.state; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.not; + +public class ClusterStateIT extends ESSingleNodeTestCase { + + public static class CustomPlugin extends Plugin { + + public CustomPlugin() { + + } + + static class CustomPluginCustom implements MetaData.Custom { + + @Override + public EnumSet context() { + return MetaData.ALL_CONTEXTS; + } + + @Override + public Diff diff(final MetaData.Custom previousState) { + return null; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + { + + } + builder.endObject(); + return builder; + } + } + + @Override + public List getNamedWriteables() { + return super.getNamedWriteables(); + } + + public static final String TYPE = "custom_plugin"; + + private final AtomicBoolean installed = new AtomicBoolean(); + + @Override + public Collection createComponents( + final Client client, + final ClusterService clusterService, + final ThreadPool threadPool, + final ResourceWatcherService resourceWatcherService, + final ScriptService scriptService, + final NamedXContentRegistry xContentRegistry, + final Environment environment, + final NodeEnvironment nodeEnvironment, + final NamedWriteableRegistry namedWriteableRegistry) { + clusterService.addListener(event -> { + final ClusterState state = event.state(); + if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) { + return; + } + + final MetaData metaData = state.metaData(); + if (state.nodes().isLocalNodeElectedMaster()) { + if (metaData.custom(CustomPlugin.TYPE) == null) { + if (installed.compareAndSet(false, true)) { + clusterService.submitStateUpdateTask("install-metadata-custom", new ClusterStateUpdateTask(Priority.URGENT) { + + @Override + public ClusterState execute(ClusterState currentState) { + if (currentState.custom(CustomPlugin.TYPE) == null) { + final MetaData.Builder builder = MetaData.builder(currentState.metaData()); + builder.putCustom(CustomPlugin.TYPE, new CustomPluginCustom()); + return ClusterState.builder(currentState).metaData(builder).build(); + } else { + return currentState; + } + } + + @Override + public void onFailure(String source, Exception e) { + throw new AssertionError(e); + } + + }); + } + } + } + + }); + return Collections.emptyList(); + } + } + + @Override + protected Collection> getPlugins() { + return Stream.concat(super.getPlugins().stream(), Stream.of(CustomPlugin.class)).collect(Collectors.toCollection(ArrayList::new)); + } + + public void testRequestCustoms() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(true).get(); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, hasItem(CustomPlugin.TYPE)); + } + + public void testDoNotRequestCustoms() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(false).get(); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, not(hasItem(CustomPlugin.TYPE))); + } + + public void testRequestCustomsDefault() { + final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).get(); + final ImmutableOpenMap customs = state.getState().metaData().customs(); + final Set keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class))); + assertThat(keys, not(hasItem(CustomPlugin.TYPE))); + } + + public void testValidation() { + final ClusterStateRequest request = new ClusterStateRequest().metaData(false).metaDataCustoms(true); + final ActionRequestValidationException e = request.validate(); + assertThat(e, hasToString(containsString("metadata customs were requested without requesting metadata"))); + } + +} diff --git a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 1dc853db59467..5061db1a33ca7 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -94,7 +94,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -303,7 +305,7 @@ public void testRestoreCustomMetadata() throws Exception { assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1)); logger.info("--> check that custom persistent metadata was restored"); - ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); + ClusterState clusterState = client.admin().cluster().prepareState().setMetaDataCustoms(true).get().getState(); logger.info("Cluster state: {}", clusterState); MetaData metaData = clusterState.getMetaData(); assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s")); @@ -316,7 +318,7 @@ public void testRestoreCustomMetadata() throws Exception { ensureYellow(); logger.info("--> check that gateway-persistent custom metadata survived full cluster restart"); - clusterState = client().admin().cluster().prepareState().get().getState(); + clusterState = client().admin().cluster().prepareState().setMetaDataCustoms(true).get().getState(); logger.info("Cluster state: {}", clusterState); metaData = clusterState.getMetaData(); assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue()); @@ -729,7 +731,10 @@ public void sendResponse(RestResponse response) { RestClusterStateAction clusterStateAction = new RestClusterStateAction(nodeSettings, mock(RestController.class), internalCluster().getInstance(SettingsFilter.class)); - RestRequest clusterStateRequest = new FakeRestRequest(); + RestRequest clusterStateRequest = + new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY) + .withParams(new HashMap<>(Collections.singletonMap("metric", "metadata,metadata_customs"))) + .build(); final CountDownLatch clusterStateLatch = new CountDownLatch(1); final AtomicReference clusterStateError = new AtomicReference<>(); clusterStateAction.handleRequest(clusterStateRequest, new AbstractRestChannel(clusterStateRequest, true) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java index 23cb579bfdc92..7cbfa5b58d614 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -40,7 +40,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -67,7 +66,8 @@ public void testRepositoryCreation() throws Exception { assertThat(FileSystemUtils.files(location).length, equalTo(numberOfFiles)); logger.info("--> check that repository is really there"); - ClusterStateResponse clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).get(); + ClusterStateResponse clusterStateResponse = + client.admin().cluster().prepareState().clear().setMetaData(true).setMetaDataCustoms(true).get(); MetaData metaData = clusterStateResponse.getState().getMetaData(); RepositoriesMetaData repositoriesMetaData = metaData.custom(RepositoriesMetaData.TYPE); assertThat(repositoriesMetaData, notNullValue()); @@ -82,7 +82,7 @@ public void testRepositoryCreation() throws Exception { assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); logger.info("--> check that both repositories are in cluster state"); - clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).get(); + clusterStateResponse = client.admin().cluster().prepareState().clear().setMetaData(true).setMetaDataCustoms(true).get(); metaData = clusterStateResponse.getState().getMetaData(); repositoriesMetaData = metaData.custom(RepositoriesMetaData.TYPE); assertThat(repositoriesMetaData, notNullValue()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java index 8e326e3556b59..7af178b588526 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.integration; import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.rest.ESRestTestCase; @@ -35,8 +36,10 @@ public void clearMlMetadata() throws IOException { @SuppressWarnings("unchecked") private void deleteAllDatafeeds() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.datafeeds"))); + final Request datafeedsRequest = new Request("GET", "/_cluster/state/metadata,metadata_customs"); + datafeedsRequest.addParameter("filter_path", "metadata.ml.datafeeds"); + final Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest(datafeedsRequest)); + List> datafeeds = (List>) XContentMapValues.extractValue("metadata.ml.datafeeds", clusterStateAsMap); if (datafeeds == null) { @@ -75,8 +78,9 @@ private void deleteAllDatafeeds() throws IOException { } private void deleteAllJobs() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.jobs"))); + final Request jobsRequest = new Request("GET", "/_cluster/state/metadata,metadata_customs"); + jobsRequest.addParameter("filter_path", "metadata.ml.jobs"); + final Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest(jobsRequest)); @SuppressWarnings("unchecked") List> jobConfigs = (List>) XContentMapValues.extractValue("metadata.ml.jobs", clusterStateAsMap); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 9ed14c2f860ef..a3dc6173d5c2f 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -549,7 +549,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} @@ -562,7 +562,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: metadata.persistent_tasks.tasks: [] @@ -790,7 +790,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} @@ -804,7 +804,7 @@ headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser cluster.state: - metric: [ metadata ] + metric: [ metadata,metadata_customs ] filter_path: metadata.persistent_tasks - match: metadata.persistent_tasks.tasks: []