Do not return metadata customs by default

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.state;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -29,11 +30,14 @@
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 public class ClusterStateRequest extends MasterNodeReadRequest<ClusterStateRequest> implements IndicesRequest.Replaceable {
 
     private boolean routingTable = true;
     private boolean nodes = true;
     private boolean metaData = true;
+    private boolean metaDataCustoms = false;
     private boolean blocks = true;
     private boolean customs = true;
     private String[] indices = Strings.EMPTY_ARRAY;
@@ -47,6 +51,9 @@ public ClusterStateRequest(StreamInput in) throws IOException {
         routingTable = in.readBoolean();
         nodes = in.readBoolean();
         metaData = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            metaDataCustoms = in.readBoolean();
+        }
         blocks = in.readBoolean();
         customs = in.readBoolean();
         indices = in.readStringArray();
@@ -59,6 +66,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(routingTable);
         out.writeBoolean(nodes);
         out.writeBoolean(metaData);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeBoolean(metaDataCustoms);
+        }
         out.writeBoolean(blocks);
         out.writeBoolean(customs);
         out.writeStringArray(indices);
@@ -67,13 +77,18 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public ActionRequestValidationException validate() {
-        return null;
+        if (metaData == false && metaDataCustoms) {
+            return addValidationError("metadata customs were requested without requesting metadata", null);
+        } else {
+            return null;
+        }
     }
 
     public ClusterStateRequest all() {
         routingTable = true;
         nodes = true;
         metaData = true;
+        metaDataCustoms = true;
         blocks = true;
         customs = true;
         indices = Strings.EMPTY_ARRAY;
@@ -84,6 +99,7 @@ public ClusterStateRequest clear() {
         routingTable = false;
         nodes = false;
         metaData = false;
+        metaDataCustoms = false;
         blocks = false;
         customs = false;
         indices = Strings.EMPTY_ARRAY;
@@ -117,6 +133,15 @@ public ClusterStateRequest metaData(boolean metaData) {
         return this;
     }
 
+    public boolean metaDataCustoms() {
+        return metaDataCustoms;
+    }
+
+    public ClusterStateRequest metaDataCustoms(boolean metaDataCustoms) {
+        this.metaDataCustoms = metaDataCustoms;
+        return this;
+    }
+
     public boolean blocks() {
         return blocks;
     }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java

@@ -59,6 +59,11 @@ public ClusterStateRequestBuilder setMetaData(boolean filter) {
         return this;
     }
 
+    public ClusterStateRequestBuilder setMetaDataCustoms(boolean filter) {
+        request.metaDataCustoms(filter);
+        return this;
+    }
+
     /**
      * Should the cluster state result include the {@link org.elasticsearch.cluster.node.DiscoveryNodes}. Defaults
      * to {@code true}.

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState;
 
@@ -115,13 +116,19 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS
                 mdBuilder = MetaData.builder(currentState.metaData());
             }
 
-            // filter out metadata that shouldn't be returned by the API
-            for (ObjectObjectCursor<String, Custom> custom : currentState.metaData().customs()) {
-                if (custom.value.context().contains(MetaData.XContentContext.API) == false) {
-                    mdBuilder.removeCustom(custom.key);
+            final Predicate<Custom> predicate;
+            if (request.metaDataCustoms()) {
+                predicate = c -> c.context().contains(MetaData.XContentContext.API);
+            } else {
+                predicate = c -> false;
+            }
+            for (final ObjectObjectCursor<String, Custom> cursor : currentState.metaData().customs()) {
+                if (predicate.test(cursor.value) == false) {
+                    mdBuilder.removeCustom(cursor.key);
                 }
             }
         }
+
         builder.metaData(mdBuilder);
 
         if (request.customs()) {

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -22,7 +22,6 @@
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -277,6 +276,7 @@ public enum Metric {
         BLOCKS("blocks"),
         NODES("nodes"),
         METADATA("metadata"),
+        METADATA_CUSTOMS("metadata_customs"),
         ROUTING_TABLE("routing_table"),
         ROUTING_NODES("routing_nodes"),
         CUSTOMS("customs");

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java

@@ -85,6 +85,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             clusterStateRequest.routingTable(
                     metrics.contains(ClusterState.Metric.ROUTING_TABLE) || metrics.contains(ClusterState.Metric.ROUTING_NODES));
             clusterStateRequest.metaData(metrics.contains(ClusterState.Metric.METADATA));
+            clusterStateRequest.metaDataCustoms(metrics.contains(ClusterState.Metric.METADATA_CUSTOMS));
             clusterStateRequest.blocks(metrics.contains(ClusterState.Metric.BLOCKS));
             clusterStateRequest.customs(metrics.contains(ClusterState.Metric.CUSTOMS));
         }

server/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateIT.java

@@ -0,0 +1,193 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.state;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.not;
+
+public class ClusterStateIT extends ESSingleNodeTestCase {
+
+    public static class CustomPlugin extends Plugin {
+
+        public CustomPlugin() {
+
+        }
+
+        static class CustomPluginCustom implements MetaData.Custom {
+
+            @Override
+            public EnumSet<MetaData.XContentContext> context() {
+                return MetaData.ALL_CONTEXTS;
+            }
+
+            @Override
+            public Diff<MetaData.Custom> diff(final MetaData.Custom previousState) {
+                return null;
+            }
+
+            @Override
+            public String getWriteableName() {
+                return TYPE;
+            }
+
+            @Override
+            public void writeTo(final StreamOutput out) throws IOException {
+
+            }
+
+            @Override
+            public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+                builder.startObject();
+                {
+
+                }
+                builder.endObject();
+                return builder;
+            }
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return super.getNamedWriteables();
+        }
+
+        public static final String TYPE = "custom_plugin";
+
+        private final AtomicBoolean installed = new AtomicBoolean();
+
+        @Override
+        public Collection<Object> createComponents(
+                final Client client,
+                final ClusterService clusterService,
+                final ThreadPool threadPool,
+                final ResourceWatcherService resourceWatcherService,
+                final ScriptService scriptService,
+                final NamedXContentRegistry xContentRegistry,
+                final Environment environment,
+                final NodeEnvironment nodeEnvironment,
+                final NamedWriteableRegistry namedWriteableRegistry) {
+            clusterService.addListener(event -> {
+                final ClusterState state = event.state();
+                if (state.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) {
+                    return;
+                }
+
+                final MetaData metaData = state.metaData();
+                if (state.nodes().isLocalNodeElectedMaster()) {
+                    if (metaData.custom(CustomPlugin.TYPE) == null) {
+                        if (installed.compareAndSet(false, true)) {
+                            clusterService.submitStateUpdateTask("install-metadata-custom", new ClusterStateUpdateTask(Priority.URGENT) {
+
+                                @Override
+                                public ClusterState execute(ClusterState currentState) {
+                                    if (currentState.custom(CustomPlugin.TYPE) == null) {
+                                        final MetaData.Builder builder = MetaData.builder(currentState.metaData());
+                                        builder.putCustom(CustomPlugin.TYPE, new CustomPluginCustom());
+                                        return ClusterState.builder(currentState).metaData(builder).build();
+                                    } else {
+                                        return currentState;
+                                    }
+                                }
+
+                                @Override
+                                public void onFailure(String source, Exception e) {
+                                    throw new AssertionError(e);
+                                }
+
+                            });
+                        }
+                    }
+                }
+
+            });
+            return Collections.emptyList();
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Stream.concat(super.getPlugins().stream(), Stream.of(CustomPlugin.class)).collect(Collectors.toCollection(ArrayList::new));
+    }
+
+    public void testRequestCustoms() {
+        final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(true).get();
+        final ImmutableOpenMap<String, MetaData.Custom> customs = state.getState().metaData().customs();
+        final Set<String> keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class)));
+        assertThat(keys, hasItem(CustomPlugin.TYPE));
+    }
+
+    public void testDoNotRequestCustoms() {
+        final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).setMetaDataCustoms(false).get();
+        final ImmutableOpenMap<String, MetaData.Custom> customs = state.getState().metaData().customs();
+        final Set<String> keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class)));
+        assertThat(keys, not(hasItem(CustomPlugin.TYPE)));
+    }
+
+    public void testRequestCustomsDefault() {
+        final ClusterStateResponse state = client().admin().cluster().prepareState().setMetaData(true).get();
+        final ImmutableOpenMap<String, MetaData.Custom> customs = state.getState().metaData().customs();
+        final Set<String> keys = new HashSet<>(Arrays.asList(customs.keys().toArray(String.class)));
+        assertThat(keys, not(hasItem(CustomPlugin.TYPE)));
+    }
+
+    public void testValidation() {
+        final ClusterStateRequest request = new ClusterStateRequest().metaData(false).metaDataCustoms(true);
+        final ActionRequestValidationException e = request.validate();
+        assertThat(e, hasToString(containsString("metadata customs were requested without requesting metadata")));
+    }
+
+}

server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java

@@ -94,7 +94,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -303,7 +305,7 @@ public void testRestoreCustomMetadata() throws Exception {
         assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1));
 
         logger.info("--> check that custom persistent metadata was restored");
-        ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
+        ClusterState clusterState = client.admin().cluster().prepareState().setMetaDataCustoms(true).get().getState();
         logger.info("Cluster state: {}", clusterState);
         MetaData metaData = clusterState.getMetaData();
         assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s"));
@@ -316,7 +318,7 @@ public void testRestoreCustomMetadata() throws Exception {
         ensureYellow();
 
         logger.info("--> check that gateway-persistent custom metadata survived full cluster restart");
-        clusterState = client().admin().cluster().prepareState().get().getState();
+        clusterState = client().admin().cluster().prepareState().setMetaDataCustoms(true).get().getState();
         logger.info("Cluster state: {}", clusterState);
         metaData = clusterState.getMetaData();
         assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue());
@@ -729,7 +731,10 @@ public void sendResponse(RestResponse response) {
 
         RestClusterStateAction clusterStateAction = new RestClusterStateAction(nodeSettings, mock(RestController.class),
                 internalCluster().getInstance(SettingsFilter.class));
-        RestRequest clusterStateRequest = new FakeRestRequest();
+        RestRequest clusterStateRequest =
+                new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+                        .withParams(new HashMap<>(Collections.singletonMap("metric", "metadata,metadata_customs")))
+                        .build();
         final CountDownLatch clusterStateLatch = new CountDownLatch(1);
         final AtomicReference<AssertionError> clusterStateError = new AtomicReference<>();
         clusterStateAction.handleRequest(clusterStateRequest, new AbstractRestChannel(clusterStateRequest, true) {