From 2a89e13e43d62ec9c443bfaba42c9a1da7e81223 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 15 Jul 2020 16:50:44 +0200 Subject: [PATCH] Move data stream transport and rest action to xpack (#59593) Backport of #59525 to 7.x branch. * Actions are moved to xpack core. * Transport and rest actions are moved the data-streams module. * Removed data streams methods from Client interface. * Adjusted tests to use client.execute(...) instead of data stream specific methods. * only attempt to delete all data streams if xpack is installed in rest tests * Now that ds apis are in xpack and ESIntegTestCase no longers deletes all ds, do that in the MlNativeIntegTestCase class for ml tests. --- .../DataStreamsStatsResponseTests.java | 2 +- .../indices/GetDataStreamResponseTests.java | 4 +- .../elasticsearch/action/ActionModule.java | 27 +- .../datastream/CreateDataStreamAction.java | 148 ------ .../datastream/DataStreamsStatsAction.java | 441 ------------------ .../datastream/DeleteDataStreamAction.java | 233 --------- .../client/IndicesAdminClient.java | 33 -- .../client/support/AbstractClient.java | 33 -- .../indices/RestCreateDataStreamAction.java | 50 -- .../indices/RestDeleteDataStreamAction.java | 50 -- .../indices/RestGetDataStreamsAction.java | 52 --- .../DeleteDataStreamRequestTests.java | 222 --------- .../GetDataStreamsRequestTests.java | 143 ------ .../GetDataStreamsResponseTests.java | 47 -- .../mapping/put/PutMappingRequestTests.java | 12 +- .../MetadataRolloverServiceTests.java | 11 +- .../metadata/DataStreamMetadataTests.java | 3 +- .../cluster/metadata/DataStreamTests.java | 22 +- .../MetadataDeleteIndexServiceTests.java | 7 +- .../MetadataIndexStateServiceTests.java | 4 +- .../cluster/metadata/MetadataTests.java | 2 +- .../cluster/DataStreamTestHelper.java | 72 ++- .../org/elasticsearch/test/TestCluster.java | 16 - .../test/rest/ESRestTestCase.java | 4 +- .../core/action/CreateDataStreamAction.java | 87 ++++ .../core/action/DataStreamsStatsAction.java | 257 ++++++++++ .../core/action/DeleteDataStreamAction.java | 102 ++++ .../core/action}/GetDataStreamAction.java | 104 +---- .../authz/privilege/IndexPrivilege.java | 6 +- .../datastreams/DataStreamIT.java | 80 ++-- .../datastreams/DataStreamsSnapshotsIT.java | 82 ++-- .../ShardClusterSnapshotRestoreIT.java | 18 +- .../xpack/datastreams/DataStreamsPlugin.java | 51 +- .../CreateDataStreamTransportAction.java | 80 ++++ .../DataStreamsStatsTransportAction.java | 217 +++++++++ .../DeleteDataStreamTransportAction.java | 160 +++++++ .../action/GetDataStreamsTransportAction.java | 119 +++++ .../rest/RestCreateDataStreamAction.java | 35 ++ .../rest}/RestDataStreamsStatsAction.java | 31 +- .../rest/RestDeleteDataStreamAction.java | 38 ++ .../rest/RestGetDataStreamsAction.java | 38 ++ .../action}/CreateDataStreamRequestTests.java | 23 +- .../DataStreamsStatsResponseTests.java | 49 +- .../action/DeleteDataStreamRequestTests.java | 42 ++ .../action/GetDataStreamsRequestTests.java | 43 ++ .../action/GetDataStreamsResponseTests.java | 40 ++ .../datastreams/DataStreamsStatsTests.java | 6 +- .../DeleteDataStreamTransportActionTests.java | 160 +++++++ .../GetDataStreamsTransportActionTests.java | 111 +++++ .../DataStreamTimestampFieldMapperTests.java | 20 - .../qa/native-multi-node-tests/build.gradle | 1 + .../ml/integration/MlNativeIntegTestCase.java | 20 +- .../security/authz/AuthorizationService.java | 2 +- 53 files changed, 1867 insertions(+), 1793 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java delete mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java delete mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java delete mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java delete mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/CreateDataStreamAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamsStatsAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java rename {server/src/main/java/org/elasticsearch/action/admin/indices/datastream => x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action}/GetDataStreamAction.java (62%) create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestCreateDataStreamAction.java rename {server/src/main/java/org/elasticsearch/rest/action/admin/indices => x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest}/RestDataStreamsStatsAction.java (62%) create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java create mode 100644 x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java rename {server/src/test/java/org/elasticsearch/action/admin/indices/datastream => x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action}/CreateDataStreamRequestTests.java (56%) rename {server/src/test/java/org/elasticsearch/action/admin/indices/datastream => x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action}/DataStreamsStatsResponseTests.java (59%) create mode 100644 x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DeleteDataStreamRequestTests.java create mode 100644 x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsRequestTests.java create mode 100644 x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsResponseTests.java create mode 100644 x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java create mode 100644 x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportActionTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/DataStreamsStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/DataStreamsStatsResponseTests.java index f95bcad1740d2..78fd777d24939 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/DataStreamsStatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/DataStreamsStatsResponseTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.client.indices; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.indices.datastream.DataStreamsStatsAction; +import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.client.AbstractResponseTestCase; import org.elasticsearch.common.unit.ByteSizeValue; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java index 6192a1c5dbf36..34b5b2af9bf74 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java @@ -19,8 +19,8 @@ package org.elasticsearch.client.indices; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction.Response.DataStreamInfo; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction.Response.DataStreamInfo; import org.elasticsearch.client.AbstractResponseTestCase; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.DataStream; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 284fc9b2c80a1..6b63816cdcb15 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -104,6 +104,7 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexAction; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.action.admin.indices.dangling.delete.DeleteDanglingIndexAction; @@ -114,10 +115,6 @@ import org.elasticsearch.action.admin.indices.dangling.import_index.TransportImportDanglingIndexAction; import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesAction; import org.elasticsearch.action.admin.indices.dangling.list.TransportListDanglingIndicesAction; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DataStreamsStatsAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; @@ -308,12 +305,10 @@ import org.elasticsearch.rest.action.admin.indices.RestAnalyzeAction; import org.elasticsearch.rest.action.admin.indices.RestClearIndicesCacheAction; import org.elasticsearch.rest.action.admin.indices.RestCloseIndexAction; -import org.elasticsearch.rest.action.admin.indices.RestCreateDataStreamAction; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; -import org.elasticsearch.rest.action.admin.indices.RestDataStreamsStatsAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteComponentTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteComposableIndexTemplateAction; -import org.elasticsearch.rest.action.admin.indices.RestDeleteDataStreamAction; +import org.elasticsearch.rest.action.admin.indices.RestDeleteComposableIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexAction; import org.elasticsearch.rest.action.admin.indices.RestDeleteIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestFlushAction; @@ -321,7 +316,7 @@ import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction; import org.elasticsearch.rest.action.admin.indices.RestGetComponentTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestGetComposableIndexTemplateAction; -import org.elasticsearch.rest.action.admin.indices.RestGetDataStreamsAction; +import org.elasticsearch.rest.action.admin.indices.RestGetComposableIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; @@ -592,6 +587,7 @@ public void reg actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class); actions.register(NodesReloadSecureSettingsAction.INSTANCE, TransportNodesReloadSecureSettingsAction.class); actions.register(AutoCreateAction.INSTANCE, AutoCreateAction.TransportAction.class); + actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class); //Indexed scripts actions.register(PutStoredScriptAction.INSTANCE, TransportPutStoredScriptAction.class); @@ -610,13 +606,6 @@ public void reg actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register); - // Data streams: - actions.register(CreateDataStreamAction.INSTANCE, CreateDataStreamAction.TransportAction.class); - actions.register(DeleteDataStreamAction.INSTANCE, DeleteDataStreamAction.TransportAction.class); - actions.register(GetDataStreamAction.INSTANCE, GetDataStreamAction.TransportAction.class); - actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class); - actions.register(DataStreamsStatsAction.INSTANCE, DataStreamsStatsAction.TransportAction.class); - // Persistent tasks: actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class); @@ -722,6 +711,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestUpgradeAction()); registerHandler.accept(new RestUpgradeStatusAction()); registerHandler.accept(new RestClearIndicesCacheAction()); + registerHandler.accept(new RestResolveIndexAction()); registerHandler.accept(new RestIndexAction()); registerHandler.accept(new CreateHandler()); @@ -774,13 +764,6 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestImportDanglingIndexAction()); registerHandler.accept(new RestDeleteDanglingIndexAction()); - // Data Stream API - registerHandler.accept(new RestCreateDataStreamAction()); - registerHandler.accept(new RestDeleteDataStreamAction()); - registerHandler.accept(new RestGetDataStreamsAction()); - registerHandler.accept(new RestResolveIndexAction()); - registerHandler.accept(new RestDataStreamsStatsAction()); - // CAT API registerHandler.accept(new RestAllocationAction()); registerHandler.accept(new RestShardsAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java deleted file mode 100644 index eab167ba5f68f..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices.datastream; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.ValidateActions; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; -import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.util.Objects; - -public class CreateDataStreamAction extends ActionType { - - public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction(); - public static final String NAME = "indices:admin/data_stream/create"; - - private CreateDataStreamAction() { - super(NAME, AcknowledgedResponse::new); - } - - public static class Request extends AcknowledgedRequest implements IndicesRequest { - - private final String name; - - public Request(String name) { - this.name = name; - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - if (Strings.hasText(name) == false) { - validationException = ValidateActions.addValidationError("name is missing", validationException); - } - return validationException; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.name = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(name); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Request request = (Request) o; - return name.equals(request.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } - - @Override - public String[] indices() { - return new String[]{name}; - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); - } - } - - public static class TransportAction extends TransportMasterNodeAction { - - private final MetadataCreateDataStreamService metadataCreateDataStreamService; - - @Inject - public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataCreateDataStreamService metadataCreateDataStreamService) { - super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); - this.metadataCreateDataStreamService = metadataCreateDataStreamService; - } - - @Override - protected String executor() { - return ThreadPool.Names.SAME; - } - - @Override - protected AcknowledgedResponse read(StreamInput in) throws IOException { - return new AcknowledgedResponse(in); - } - - @Override - protected void masterOperation(Request request, ClusterState state, - ActionListener listener) throws Exception { - CreateDataStreamClusterStateUpdateRequest updateRequest = new CreateDataStreamClusterStateUpdateRequest( - request.name, - request.masterNodeTimeout(), - request.timeout() - ); - metadataCreateDataStreamService.createDataStream(updateRequest, listener); - } - - @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); - } - } - -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsAction.java deleted file mode 100644 index 2ca3a9c19c179..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsAction.java +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.action.admin.indices.datastream; - -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.PointValues; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexAbstractionResolver; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardsIterator; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.ShardNotFoundException; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedMap; -import java.util.stream.Stream; - -public class DataStreamsStatsAction extends ActionType { - - public static final DataStreamsStatsAction INSTANCE = new DataStreamsStatsAction(); - public static final String NAME = "indices:monitor/data_stream/stats"; - - public DataStreamsStatsAction() { - super(NAME, DataStreamsStatsAction.Response::new); - } - - public static class Request extends BroadcastRequest { - public Request() { - super((String[]) null); - } - - public Request(StreamInput in) throws IOException { - super(in); - } - } - - public static class Response extends BroadcastResponse { - private final int dataStreamCount; - private final int backingIndices; - private final ByteSizeValue totalStoreSize; - private final DataStreamStats[] dataStreams; - - public Response(int totalShards, int successfulShards, int failedShards, List shardFailures, - int dataStreamCount, int backingIndices, ByteSizeValue totalStoreSize, DataStreamStats[] dataStreams) { - super(totalShards, successfulShards, failedShards, shardFailures); - this.dataStreamCount = dataStreamCount; - this.backingIndices = backingIndices; - this.totalStoreSize = totalStoreSize; - this.dataStreams = dataStreams; - } - - public Response(StreamInput in) throws IOException { - super(in); - this.dataStreamCount = in.readVInt(); - this.backingIndices = in.readVInt(); - this.totalStoreSize = new ByteSizeValue(in); - this.dataStreams = in.readArray(DataStreamStats::new, DataStreamStats[]::new); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeVInt(dataStreamCount); - out.writeVInt(backingIndices); - totalStoreSize.writeTo(out); - out.writeArray(dataStreams); - } - - @Override - protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { - builder.field("data_stream_count", dataStreamCount); - builder.field("backing_indices", backingIndices); - builder.humanReadableField("total_store_size_bytes", "total_store_size", totalStoreSize); - builder.array("data_streams", (Object[]) dataStreams); - } - - public int getDataStreamCount() { - return dataStreamCount; - } - - public int getBackingIndices() { - return backingIndices; - } - - public ByteSizeValue getTotalStoreSize() { - return totalStoreSize; - } - - public DataStreamStats[] getDataStreams() { - return dataStreams; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - Response response = (Response) obj; - return dataStreamCount == response.dataStreamCount && - backingIndices == response.backingIndices && - Objects.equals(totalStoreSize, response.totalStoreSize) && - Arrays.equals(dataStreams, response.dataStreams); - } - - @Override - public int hashCode() { - int result = Objects.hash(dataStreamCount, backingIndices, totalStoreSize); - result = 31 * result + Arrays.hashCode(dataStreams); - return result; - } - - @Override - public String toString() { - return "Response{" + - "dataStreamCount=" + dataStreamCount + - ", backingIndices=" + backingIndices + - ", totalStoreSize=" + totalStoreSize + - ", dataStreams=" + Arrays.toString(dataStreams) + - '}'; - } - } - - public static class DataStreamStats implements ToXContentObject, Writeable { - private final String dataStream; - private final int backingIndices; - private final ByteSizeValue storeSize; - private final long maximumTimestamp; - - public DataStreamStats(String dataStream, int backingIndices, ByteSizeValue storeSize, long maximumTimestamp) { - this.dataStream = dataStream; - this.backingIndices = backingIndices; - this.storeSize = storeSize; - this.maximumTimestamp = maximumTimestamp; - } - - public DataStreamStats(StreamInput in) throws IOException { - this.dataStream = in.readString(); - this.backingIndices = in.readVInt(); - this.storeSize = new ByteSizeValue(in); - this.maximumTimestamp = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(dataStream); - out.writeVInt(backingIndices); - storeSize.writeTo(out); - out.writeVLong(maximumTimestamp); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("data_stream", dataStream); - builder.field("backing_indices", backingIndices); - builder.humanReadableField("store_size_bytes", "store_size", storeSize); - builder.field("maximum_timestamp", maximumTimestamp); - builder.endObject(); - return builder; - } - - public String getDataStream() { - return dataStream; - } - - public int getBackingIndices() { - return backingIndices; - } - - public ByteSizeValue getStoreSize() { - return storeSize; - } - - public long getMaximumTimestamp() { - return maximumTimestamp; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - DataStreamStats that = (DataStreamStats) obj; - return backingIndices == that.backingIndices && - maximumTimestamp == that.maximumTimestamp && - Objects.equals(dataStream, that.dataStream) && - Objects.equals(storeSize, that.storeSize); - } - - @Override - public int hashCode() { - return Objects.hash(dataStream, backingIndices, storeSize, maximumTimestamp); - } - - @Override - public String toString() { - return "DataStreamStats{" + - "dataStream='" + dataStream + '\'' + - ", backingIndices=" + backingIndices + - ", storeSize=" + storeSize + - ", maximumTimestamp=" + maximumTimestamp + - '}'; - } - } - - public static class DataStreamShardStats implements Writeable { - private final ShardRouting shardRouting; - private final StoreStats storeStats; - private final long maxTimestamp; - - public DataStreamShardStats(ShardRouting shardRouting, StoreStats storeStats, long maxTimestamp) { - this.shardRouting = shardRouting; - this.storeStats = storeStats; - this.maxTimestamp = maxTimestamp; - } - - public DataStreamShardStats(StreamInput in) throws IOException { - this.shardRouting = new ShardRouting(in); - this.storeStats = new StoreStats(in); - this.maxTimestamp = in.readVLong(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - shardRouting.writeTo(out); - storeStats.writeTo(out); - out.writeVLong(maxTimestamp); - } - - public ShardRouting getShardRouting() { - return shardRouting; - } - - public StoreStats getStoreStats() { - return storeStats; - } - - public long getMaxTimestamp() { - return maxTimestamp; - } - } - - private static class AggregatedStats { - Set backingIndices = new HashSet<>(); - long storageBytes = 0L; - long maxTimestamp = 0L; - } - - public static class TransportAction extends TransportBroadcastByNodeAction { - - private final ClusterService clusterService; - private final IndicesService indicesService; - private final IndexAbstractionResolver indexAbstractionResolver; - - @Inject - public TransportAction(ClusterService clusterService, TransportService transportService, IndicesService indicesService, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(DataStreamsStatsAction.NAME, clusterService, transportService, actionFilters, indexNameExpressionResolver, - Request::new, ThreadPool.Names.MANAGEMENT); - this.clusterService = clusterService; - this.indicesService = indicesService; - this.indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); - } - - @Override - protected Request readRequestFrom(StreamInput in) throws IOException { - return new Request(in); - } - - @Override - protected ClusterBlockException checkGlobalBlock(ClusterState state, Request request) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); - } - - @Override - protected ClusterBlockException checkRequestBlock(ClusterState state, Request request, String[] concreteIndices) { - return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); - } - - @Override - protected ShardsIterator shards(ClusterState clusterState, Request request, String[] concreteIndices) { - String[] requestIndices = request.indices(); - if (requestIndices == null || requestIndices.length == 0) { - requestIndices = new String[]{"*"}; - } - List abstractionNames = indexAbstractionResolver.resolveIndexAbstractions(requestIndices, request.indicesOptions(), - clusterState.getMetadata(), true); // Always include data streams for data streams stats api - SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); - - String[] concreteDatastreamIndices = abstractionNames.stream().flatMap(abstractionName -> { - IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName); - assert indexAbstraction != null; - if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { - IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) indexAbstraction; - List indices = dataStream.getIndices(); - return indices.stream().map(idx -> idx.getIndex().getName()); - } else { - return Stream.empty(); - } - }).toArray(String[]::new); - return clusterState.getRoutingTable().allShards(concreteDatastreamIndices); - } - - @Override - protected DataStreamShardStats shardOperation(Request request, ShardRouting shardRouting) throws IOException { - IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); - // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet - if (indexShard.routingEntry() == null) { - throw new ShardNotFoundException(indexShard.shardId()); - } - StoreStats storeStats = indexShard.storeStats(); - IndexAbstraction indexAbstraction = clusterService.state().getMetadata().getIndicesLookup().get(shardRouting.getIndexName()); - assert indexAbstraction != null; - IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); - assert dataStream != null; - long maxTimestamp = 0L; - try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { - IndexReader indexReader = searcher.getIndexReader(); - String fieldName = dataStream.getDataStream().getTimeStampField().getName(); - byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, fieldName); - if (maxPackedValue != null) { - maxTimestamp = LongPoint.decodeDimension(maxPackedValue, 0); - } - } - return new DataStreamShardStats( - indexShard.routingEntry(), - storeStats, - maxTimestamp - ); - } - - @Override - protected DataStreamShardStats readShardResult(StreamInput in) throws IOException { - return new DataStreamShardStats(in); - } - - @Override - protected Response newResponse(Request request, int totalShards, int successfulShards, - int failedShards, List dataStreamShardStats, - List shardFailures, - ClusterState clusterState) { - Map aggregatedDataStreamsStats = new HashMap<>(); - Set allBackingIndices = new HashSet<>(); - long totalStoreSizeBytes = 0L; - - SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); - for (DataStreamShardStats shardStat : dataStreamShardStats) { - String indexName = shardStat.getShardRouting().getIndexName(); - IndexAbstraction indexAbstraction = indicesLookup.get(indexName); - IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); - assert dataStream != null; - - // Aggregate global stats - totalStoreSizeBytes += shardStat.getStoreStats().sizeInBytes(); - allBackingIndices.add(indexName); - - // Aggregate data stream stats - AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats()); - stats.storageBytes += shardStat.getStoreStats().sizeInBytes(); - stats.maxTimestamp = Math.max(stats.maxTimestamp, shardStat.getMaxTimestamp()); - stats.backingIndices.add(indexName); - } - - DataStreamStats[] dataStreamStats = aggregatedDataStreamsStats.entrySet().stream() - .map(entry -> new DataStreamStats( - entry.getKey(), - entry.getValue().backingIndices.size(), - new ByteSizeValue(entry.getValue().storageBytes), - entry.getValue().maxTimestamp)) - .toArray(DataStreamStats[]::new); - - return new Response( - totalShards, - successfulShards, - failedShards, - shardFailures, - aggregatedDataStreamsStats.size(), - allBackingIndices.size(), - new ByteSizeValue(totalStoreSizeBytes), - dataStreamStats - ); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java deleted file mode 100644 index 7a0b93ff44639..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices.datastream; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.index.Index; -import org.elasticsearch.snapshots.SnapshotInProgressException; -import org.elasticsearch.snapshots.SnapshotsService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -import static org.elasticsearch.action.ValidateActions.addValidationError; - -public class DeleteDataStreamAction extends ActionType { - - private static final Logger logger = LogManager.getLogger(DeleteDataStreamAction.class); - - public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction(); - public static final String NAME = "indices:admin/data_stream/delete"; - - private DeleteDataStreamAction() { - super(NAME, AcknowledgedResponse::new); - } - - public static class Request extends MasterNodeRequest implements IndicesRequest.Replaceable { - - private String[] names; - - public Request(String[] names) { - this.names = Objects.requireNonNull(names); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - if (CollectionUtils.isEmpty(names)) { - validationException = addValidationError("no data stream(s) specified", validationException); - } - return validationException; - } - - public Request(StreamInput in) throws IOException { - super(in); - this.names = in.readStringArray(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeStringArray(names); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Request request = (Request) o; - return Arrays.equals(names, request.names); - } - - @Override - public int hashCode() { - return Arrays.hashCode(names); - } - - @Override - public String[] indices() { - return names; - } - - @Override - public IndicesOptions indicesOptions() { - // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and - // a data stream's backing indices are retrieved from its metadata - return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); - } - - @Override - public boolean includeDataStreams() { - return true; - } - - @Override - public IndicesRequest indices(String... indices) { - this.names = indices; - return this; - } - } - - public static class TransportAction extends TransportMasterNodeAction { - - private final MetadataDeleteIndexService deleteIndexService; - - @Inject - public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - MetadataDeleteIndexService deleteIndexService) { - super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); - this.deleteIndexService = deleteIndexService; - } - - @Override - protected String executor() { - return ThreadPool.Names.SAME; - } - - @Override - protected AcknowledgedResponse read(StreamInput in) throws IOException { - return new AcknowledgedResponse(in); - } - - @Override - protected void masterOperation(Request request, ClusterState state, - ActionListener listener) throws Exception { - clusterService.submitStateUpdateTask("remove-data-stream [" + Strings.arrayToCommaDelimitedString(request.names) + "]", - new ClusterStateUpdateTask(Priority.HIGH) { - - @Override - public TimeValue timeout() { - return request.masterNodeTimeout(); - } - - @Override - public void onFailure(String source, Exception e) { - listener.onFailure(e); - } - - @Override - public ClusterState execute(ClusterState currentState) { - return removeDataStream(deleteIndexService, currentState, request); - } - - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - listener.onResponse(new AcknowledgedResponse(true)); - } - }); - } - - static ClusterState removeDataStream(MetadataDeleteIndexService deleteIndexService, ClusterState currentState, Request request) { - Set dataStreams = new HashSet<>(); - Set snapshottingDataStreams = new HashSet<>(); - for (String name : request.names) { - for (String dataStreamName : currentState.metadata().dataStreams().keySet()) { - if (Regex.simpleMatch(name, dataStreamName)) { - dataStreams.add(dataStreamName); - } - } - - snapshottingDataStreams.addAll(SnapshotsService.snapshottingDataStreams(currentState, dataStreams)); - } - - if (snapshottingDataStreams.isEmpty() == false) { - throw new SnapshotInProgressException("Cannot delete data streams that are being snapshotted: " + snapshottingDataStreams + - ". Try again after snapshot finishes or cancel the currently running snapshot."); - } - - Set backingIndicesToRemove = new HashSet<>(); - for (String dataStreamName : dataStreams) { - DataStream dataStream = currentState.metadata().dataStreams().get(dataStreamName); - assert dataStream != null; - backingIndicesToRemove.addAll(dataStream.getIndices()); - } - - // first delete the data streams and then the indices: - // (this to avoid data stream validation from failing when deleting an index that is part of a data stream - // without updating the data stream) - // TODO: change order when delete index api also updates the data stream the index to be removed is member of - Metadata.Builder metadata = Metadata.builder(currentState.metadata()); - for (String ds : dataStreams) { - logger.info("removing data stream [{}]", ds); - metadata.removeDataStream(ds); - } - currentState = ClusterState.builder(currentState).metadata(metadata).build(); - return deleteIndexService.deleteIndices(currentState, backingIndicesToRemove); - } - - @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); - } - } - -} diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index f63ecd4d39e7e..e73f6a1cf481a 100644 --- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -39,9 +39,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; @@ -844,36 +841,6 @@ public interface IndicesAdminClient extends ElasticsearchClient { */ void rolloverIndex(RolloverRequest request, ActionListener listener); - /** - * Store a data stream - */ - void createDataStream(CreateDataStreamAction.Request request, ActionListener listener); - - /** - * Store a data stream - */ - ActionFuture createDataStream(CreateDataStreamAction.Request request); - - /** - * Delete a data stream - */ - void deleteDataStream(DeleteDataStreamAction.Request request, ActionListener listener); - - /** - * Delete a data stream - */ - ActionFuture deleteDataStream(DeleteDataStreamAction.Request request); - - /** - * Get data streams - */ - void getDataStreams(GetDataStreamAction.Request request, ActionListener listener); - - /** - * Get data streams - */ - ActionFuture getDataStreams(GetDataStreamAction.Request request); - /** * Resolves names and wildcard expressions to indices, aliases, and data streams */ diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index daab9926ef945..879c60d065166 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -166,9 +166,6 @@ import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesAction; import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesRequest; import org.elasticsearch.action.admin.indices.dangling.list.ListDanglingIndicesResponse; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; @@ -1799,36 +1796,6 @@ public void getSettings(GetSettingsRequest request, ActionListener listener) { - execute(CreateDataStreamAction.INSTANCE, request, listener); - } - - @Override - public ActionFuture createDataStream(CreateDataStreamAction.Request request) { - return execute(CreateDataStreamAction.INSTANCE, request); - } - - @Override - public void deleteDataStream(DeleteDataStreamAction.Request request, ActionListener listener) { - execute(DeleteDataStreamAction.INSTANCE, request, listener); - } - - @Override - public ActionFuture deleteDataStream(DeleteDataStreamAction.Request request) { - return execute(DeleteDataStreamAction.INSTANCE, request); - } - - @Override - public void getDataStreams(GetDataStreamAction.Request request, ActionListener listener) { - execute(GetDataStreamAction.INSTANCE, request, listener); - } - - @Override - public ActionFuture getDataStreams(GetDataStreamAction.Request request) { - return execute(GetDataStreamAction.INSTANCE, request); - } - @Override public void resolveIndex(ResolveIndexAction.Request request, ActionListener listener) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java deleted file mode 100644 index 94dd6c9f0d0cc..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateDataStreamAction.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -public class RestCreateDataStreamAction extends BaseRestHandler { - - @Override - public String getName() { - return "create_data_stream_action"; - } - - @Override - public List routes() { - return Collections.singletonList( - new Route(RestRequest.Method.PUT, "/_data_stream/{name}") - ); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(request.param("name")); - return channel -> client.admin().indices().createDataStream(putDataStreamRequest, new RestToXContentListener<>(channel)); - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java deleted file mode 100644 index f963444ecb6a0..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteDataStreamAction.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -public class RestDeleteDataStreamAction extends BaseRestHandler { - - @Override - public String getName() { - return "delete_data_stream_action"; - } - - @Override - public List routes() { - return Collections.singletonList(new Route(RestRequest.Method.DELETE, "/_data_stream/{name}")); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request( - Strings.splitStringByCommaToArray(request.param("name"))); - return channel -> client.admin().indices().deleteDataStream(deleteDataStreamRequest, new RestToXContentListener<>(channel)); - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java deleted file mode 100644 index 569715c0e5e13..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.List; - -public class RestGetDataStreamsAction extends BaseRestHandler { - - @Override - public String getName() { - return "get_data_streams_action"; - } - - @Override - public List routes() { - return org.elasticsearch.common.collect.List.of( - new Route(RestRequest.Method.GET, "/_data_stream"), - new Route(RestRequest.Method.GET, "/_data_stream/{name}") - ); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request( - Strings.splitStringByCommaToArray(request.param("name"))); - return channel -> client.admin().indices().getDataStreams(getDataStreamsRequest, new RestToXContentListener<>(channel)); - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java deleted file mode 100644 index a899b97a65e9b..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices.datastream; - -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction.Request; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; -import org.elasticsearch.snapshots.Snapshot; -import org.elasticsearch.snapshots.SnapshotId; -import org.elasticsearch.snapshots.SnapshotInProgressException; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return Request::new; - } - - @Override - protected Request createTestInstance() { - return new Request(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(6))); - } - - public void testValidateRequest() { - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{"my-data-stream"}); - ActionRequestValidationException e = req.validate(); - assertNull(e); - } - - public void testValidateRequestWithoutName() { - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[0]); - ActionRequestValidationException e = req.validate(); - assertNotNull(e); - assertThat(e.validationErrors().size(), equalTo(1)); - assertThat(e.validationErrors().get(0), containsString("no data stream(s) specified")); - } - - public void testDeleteDataStream() { - final String dataStreamName = "my-data-stream"; - final List otherIndices = randomSubsetOf(org.elasticsearch.common.collect.List.of("foo", "bar", "baz")); - ClusterState cs = getClusterStateWithDataStreams( - org.elasticsearch.common.collect.List.of(new Tuple<>(dataStreamName, 2)), otherIndices); - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{dataStreamName}); - ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); - assertThat(newState.metadata().dataStreams().size(), equalTo(0)); - assertThat(newState.metadata().indices().size(), equalTo(otherIndices.size())); - for (String indexName : otherIndices) { - assertThat(newState.metadata().indices().get(indexName).getIndex().getName(), equalTo(indexName)); - } - } - - public void testDeleteMultipleDataStreams() { - String[] dataStreamNames = {"foo", "bar", "baz", "eggplant"}; - ClusterState cs = getClusterStateWithDataStreams(org.elasticsearch.common.collect.List.of( - new Tuple<>(dataStreamNames[0], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[1], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[2], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[3], randomIntBetween(1, 3)) - ), org.elasticsearch.common.collect.List.of()); - - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{"ba*", "eggplant"}); - ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); - assertThat(newState.metadata().dataStreams().size(), equalTo(1)); - DataStream remainingDataStream = newState.metadata().dataStreams().get(dataStreamNames[0]); - assertNotNull(remainingDataStream); - assertThat(newState.metadata().indices().size(), equalTo(remainingDataStream.getIndices().size())); - for (Index i : remainingDataStream.getIndices()) { - assertThat(newState.metadata().indices().get(i.getName()).getIndex(), equalTo(i)); - } - } - - public void testDeleteSnapshottingDataStream() { - final String dataStreamName = "my-data-stream1"; - final String dataStreamName2 = "my-data-stream2"; - final List otherIndices = randomSubsetOf(Arrays.asList("foo", "bar", "baz")); - - ClusterState cs = getClusterStateWithDataStreams(Arrays.asList(new Tuple<>(dataStreamName, 2), new Tuple<>(dataStreamName2, 2)), - otherIndices); - SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.of(Arrays.asList( - createEntry(dataStreamName, "repo1", false), - createEntry(dataStreamName2, "repo2", true))); - ClusterState snapshotCs = ClusterState.builder(cs).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).build(); - - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{dataStreamName}); - SnapshotInProgressException e = expectThrows(SnapshotInProgressException.class, - () -> DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), snapshotCs, req)); - - assertThat(e.getMessage(), equalTo("Cannot delete data streams that are being snapshotted: [my-data-stream1]. Try again after " + - "snapshot finishes or cancel the currently running snapshot.")); - } - - private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo, boolean partial) { - return new SnapshotsInProgress.Entry(new Snapshot(repo, new SnapshotId("", "")), false, partial, - SnapshotsInProgress.State.STARTED, Collections.emptyList(), Collections.singletonList(dataStreamName), 0, 1, - ImmutableOpenMap.of(), null, null, null); - } - - public void testDeleteNonexistentDataStream() { - final String dataStreamName = "my-data-stream"; - String[] dataStreamNames = {"foo", "bar", "baz", "eggplant"}; - ClusterState cs = getClusterStateWithDataStreams(org.elasticsearch.common.collect.List.of( - new Tuple<>(dataStreamNames[0], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[1], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[2], randomIntBetween(1, 3)), - new Tuple<>(dataStreamNames[3], randomIntBetween(1, 3)) - ), org.elasticsearch.common.collect.List.of()); - DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{dataStreamName}); - ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); - assertThat(newState.metadata().dataStreams().size(), equalTo(cs.metadata().dataStreams().size())); - assertThat(newState.metadata().dataStreams().keySet(), - containsInAnyOrder(cs.metadata().dataStreams().keySet().toArray(Strings.EMPTY_ARRAY))); - } - - @SuppressWarnings("unchecked") - private static MetadataDeleteIndexService getMetadataDeleteIndexService() { - MetadataDeleteIndexService s = mock(MetadataDeleteIndexService.class); - when(s.deleteIndices(any(ClusterState.class), any(Set.class))) - .thenAnswer(mockInvocation -> { - ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; - Set indices = (Set) mockInvocation.getArguments()[1]; - - final Metadata.Builder b = Metadata.builder(currentState.metadata()); - for (Index index : indices) { - b.remove(index.getName()); - } - - return ClusterState.builder(currentState).metadata(b.build()).build(); - }); - - return s; - } - - /** - * Constructs {@code ClusterState} with the specified data streams and indices. - * - * @param dataStreams The names of the data streams to create with their respective number of backing indices - * @param indexNames The names of indices to create that do not back any data streams - */ - public static ClusterState getClusterStateWithDataStreams(List> dataStreams, List indexNames) { - Metadata.Builder builder = Metadata.builder(); - - List allIndices = new ArrayList<>(); - for (Tuple dsTuple : dataStreams) { - List backingIndices = new ArrayList<>(); - for (int backingIndexNumber = 1; backingIndexNumber <= dsTuple.v2(); backingIndexNumber++) { - backingIndices.add(createIndexMetadata(DataStream.getDefaultBackingIndexName(dsTuple.v1(), backingIndexNumber), true)); - } - allIndices.addAll(backingIndices); - - DataStream ds = new DataStream(dsTuple.v1(), createTimestampField("@timestamp"), - backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()), dsTuple.v2()); - builder.put(ds); - } - - for (String indexName : indexNames) { - allIndices.add(createIndexMetadata(indexName, false)); - } - - for (IndexMetadata index : allIndices) { - builder.put(index, false); - } - - return ClusterState.builder(new ClusterName("_name")).metadata(builder).build(); - } - - private static IndexMetadata createIndexMetadata(String name, boolean hidden) { - Settings.Builder b = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put("index.hidden", hidden); - - return IndexMetadata.builder(name) - .settings(b) - .numberOfShards(1) - .numberOfReplicas(1) - .build(); - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java deleted file mode 100644 index c0a29e00bf05a..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices.datastream; - -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction.Request; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.util.List; - -import static org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamRequestTests.getClusterStateWithDataStreams; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return Request::new; - } - - @Override - protected Request createTestInstance() { - final String[] searchParameter; - switch (randomIntBetween(1, 4)) { - case 1: - searchParameter = generateRandomStringArray(3, 8, false, false); - break; - case 2: - String[] parameters = generateRandomStringArray(3, 8, false, false); - for (int k = 0; k < parameters.length; k++) { - parameters[k] = parameters[k] + "*"; - } - searchParameter = parameters; - break; - case 3: - searchParameter = new String[]{"*"}; - break; - default: - searchParameter = null; - break; - } - return new Request(searchParameter); - } - - public void testGetDataStream() { - final String dataStreamName = "my-data-stream"; - ClusterState cs = getClusterStateWithDataStreams( - org.elasticsearch.common.collect.List.of(new Tuple<>(dataStreamName, 1)), org.elasticsearch.common.collect.List.of()); - GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamName}); - List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(1)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamName)); - } - - public void testGetDataStreamsWithWildcards() { - final String[] dataStreamNames = {"my-data-stream", "another-data-stream"}; - ClusterState cs = getClusterStateWithDataStreams( - org.elasticsearch.common.collect.List.of(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), - org.elasticsearch.common.collect.List.of()); - - GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamNames[1].substring(0, 5) + "*"}); - List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(1)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - - req = new GetDataStreamAction.Request(new String[]{"*"}); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(2)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); - - req = new GetDataStreamAction.Request((String[]) null); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(2)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); - - req = new GetDataStreamAction.Request(new String[]{"matches-none*"}); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(0)); - } - - public void testGetDataStreamsWithoutWildcards() { - final String[] dataStreamNames = {"my-data-stream", "another-data-stream"}; - ClusterState cs = getClusterStateWithDataStreams( - org.elasticsearch.common.collect.List.of(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), - org.elasticsearch.common.collect.List.of()); - - GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamNames[0], dataStreamNames[1]}); - List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(2)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); - - req = new GetDataStreamAction.Request(new String[]{dataStreamNames[1]}); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(1)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - - req = new GetDataStreamAction.Request(new String[]{dataStreamNames[0]}); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); - assertThat(dataStreams.size(), equalTo(1)); - assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[0])); - - GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(new String[]{"foo"}); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, - () -> GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req2)); - assertThat(e.getMessage(), containsString("no such index [foo]")); - } - - public void testGetNonexistentDataStream() { - final String dataStreamName = "my-data-stream"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamName}); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, - () -> GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req)); - assertThat(e.getMessage(), containsString("no such index [" + dataStreamName + "]")); - } - -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java deleted file mode 100644 index e90924d3ddfa2..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsResponseTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.admin.indices.datastream; - -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction.Response; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.DataStreamTests; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.util.ArrayList; -import java.util.List; - -public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return Response::new; - } - - @Override - protected Response createTestInstance() { - int numDataStreams = randomIntBetween(0, 8); - List dataStreams = new ArrayList<>(); - for (int i = 0; i < numDataStreams; i++) { - dataStreams.add(new Response.DataStreamInfo(DataStreamTests.randomInstance(), ClusterHealthStatus.GREEN, - randomAlphaOfLengthBetween(2, 10), randomAlphaOfLengthBetween(2, 10))); - } - return new Response(dataStreams); - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java index c11f9e06f0a08..0963d3259e04f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.action.admin.indices.mapping.put; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamRequestTests; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -161,7 +161,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamsAndWriteAliases() tuple(dataStreamNames[1], randomIntBetween(1, 3)), tuple(dataStreamNames[2], randomIntBetween(1, 3))); - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dsMetadata, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dsMetadata, org.elasticsearch.common.collect.List.of("index1", "index2", "index3")); cs = addAliases(cs, org.elasticsearch.common.collect.List.of( tuple("alias1", org.elasticsearch.common.collect.List.of(tuple("index1", false), tuple("index2", true))), @@ -182,7 +182,7 @@ public void testResolveIndicesWithoutWriteIndexOnlyAndDataStreamsAndWriteAliases tuple(dataStreamNames[1], randomIntBetween(1, 3)), tuple(dataStreamNames[2], randomIntBetween(1, 3))); - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dsMetadata, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dsMetadata, org.elasticsearch.common.collect.List.of("index1", "index2", "index3")); cs = addAliases(cs, org.elasticsearch.common.collect.List.of( tuple("alias1", org.elasticsearch.common.collect.List.of(tuple("index1", false), tuple("index2", true))), @@ -205,7 +205,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndDataStreamAndIndex() { tuple(dataStreamNames[1], randomIntBetween(1, 3)), tuple(dataStreamNames[2], randomIntBetween(1, 3))); - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dsMetadata, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dsMetadata, org.elasticsearch.common.collect.List.of("index1", "index2", "index3")); cs = addAliases(cs, org.elasticsearch.common.collect.List.of( tuple("alias1", org.elasticsearch.common.collect.List.of(tuple("index1", false), tuple("index2", true))), @@ -228,7 +228,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndNoSingleWriteIndex() { tuple(dataStreamNames[1], randomIntBetween(1, 3)), tuple(dataStreamNames[2], randomIntBetween(1, 3))); - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dsMetadata, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dsMetadata, org.elasticsearch.common.collect.List.of("index1", "index2", "index3")); final ClusterState cs2 = addAliases(cs, org.elasticsearch.common.collect.List.of( tuple("alias1", org.elasticsearch.common.collect.List.of(tuple("index1", false), tuple("index2", true))), @@ -247,7 +247,7 @@ public void testResolveIndicesWithWriteIndexOnlyAndAliasWithoutWriteIndex() { tuple(dataStreamNames[1], randomIntBetween(1, 3)), tuple(dataStreamNames[2], randomIntBetween(1, 3))); - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dsMetadata, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dsMetadata, org.elasticsearch.common.collect.List.of("index1", "index2", "index3")); final ClusterState cs2 = addAliases(cs, org.elasticsearch.common.collect.List.of( tuple("alias1", org.elasticsearch.common.collect.List.of(tuple("index1", false), tuple("index2", false))), diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index b40dfe9039fc6..292b950836a45 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamTests; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -258,7 +257,7 @@ public void testAliasValidation() { public void testDataStreamValidation() throws IOException { Metadata.Builder md = Metadata.builder(); - DataStream randomDataStream = DataStreamTests.randomInstance(); + DataStream randomDataStream = DataStreamTestHelper.randomInstance(); for (Index index : randomDataStream.getIndices()) { md.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); } @@ -332,7 +331,7 @@ public void testCreateIndexRequest() { } public void testCreateIndexRequestForDataStream() { - DataStream dataStream = DataStreamTests.randomInstance(); + DataStream dataStream = DataStreamTestHelper.randomInstance(); final String newWriteIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); final RolloverRequest rolloverRequest = new RolloverRequest(dataStream.getName(), randomAlphaOfLength(10)); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; @@ -537,7 +536,7 @@ public void testRolloverClusterState() throws Exception { } public void testRolloverClusterStateForDataStream() throws Exception { - final DataStream dataStream = DataStreamTests.randomInstance(); + final DataStream dataStream = DataStreamTestHelper.randomInstance(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"), null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate()); Metadata.Builder builder = Metadata.builder(); @@ -633,7 +632,7 @@ public void testValidation() throws Exception { final boolean useDataStream = randomBoolean(); final Metadata.Builder builder = Metadata.builder(); if (useDataStream) { - DataStream dataStream = DataStreamTests.randomInstance(); + DataStream dataStream = DataStreamTestHelper.randomInstance(); rolloverTarget = dataStream.getName(); sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName(); defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); @@ -690,7 +689,7 @@ public void testValidation() throws Exception { } public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { - final DataStream dataStream = DataStreamTests.randomInstance(); + final DataStream dataStream = DataStreamTestHelper.randomInstance(); Metadata.Builder builder = Metadata.builder(); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java index 7ef4aad1a1802..46c3210a49529 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamMetadataTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.test.AbstractNamedWriteableTestCase; @@ -36,7 +37,7 @@ protected DataStreamMetadata createTestInstance() { } Map dataStreams = new HashMap<>(); for (int i = 0; i < randomIntBetween(1, 5); i++) { - dataStreams.put(randomAlphaOfLength(5), DataStreamTests.randomInstance()); + dataStreams.put(randomAlphaOfLength(5), DataStreamTestHelper.randomInstance()); } return new DataStreamMetadata(dataStreams); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 0bccae724613e..826d5940245cd 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.cluster.metadata; +import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,23 +36,6 @@ public class DataStreamTests extends AbstractSerializingTestCase { - public static List randomIndexInstances() { - int numIndices = randomIntBetween(0, 128); - List indices = new ArrayList<>(numIndices); - for (int i = 0; i < numIndices; i++) { - indices.add(new Index(randomAlphaOfLength(10).toLowerCase(Locale.ROOT), UUIDs.randomBase64UUID(random()))); - } - return indices; - } - - public static DataStream randomInstance() { - List indices = randomIndexInstances(); - long generation = indices.size() + randomLongBetween(1, 128); - String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random()))); - return new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation); - } - @Override protected DataStream doParseInstance(XContentParser parser) throws IOException { return DataStream.fromXContent(parser); @@ -64,11 +48,11 @@ protected Writeable.Reader instanceReader() { @Override protected DataStream createTestInstance() { - return randomInstance(); + return DataStreamTestHelper.randomInstance(); } public void testRollover() { - DataStream ds = randomInstance(); + DataStream ds = DataStreamTestHelper.randomInstance(); Index newWriteIndex = new Index(getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1), UUIDs.randomBase64UUID(random())); DataStream rolledDs = ds.rollover(newWriteIndex); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index 91cbd75bd9a14..eca8a9a851093 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ - package org.elasticsearch.cluster.metadata; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamRequestTests; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.routing.RoutingTable; @@ -115,7 +114,7 @@ public void testDeleteUnassigned() { public void testDeleteBackingIndexForDataStream() { int numBackingIndices = randomIntBetween(2, 5); String dataStreamName = randomAlphaOfLength(6).toLowerCase(Locale.ROOT); - ClusterState before = DeleteDataStreamRequestTests.getClusterStateWithDataStreams( + ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams( List.of(new Tuple<>(dataStreamName, numBackingIndices)), List.of()); int numIndexToDelete = randomIntBetween(1, numBackingIndices - 1); @@ -132,7 +131,7 @@ public void testDeleteBackingIndexForDataStream() { public void testDeleteCurrentWriteIndexForDataStream() { int numBackingIndices = randomIntBetween(1, 5); String dataStreamName = randomAlphaOfLength(6).toLowerCase(Locale.ROOT); - ClusterState before = DeleteDataStreamRequestTests.getClusterStateWithDataStreams( + ClusterState before = DataStreamTestHelper.getClusterStateWithDataStreams( List.of(new Tuple<>(dataStreamName, numBackingIndices)), List.of()); Index indexToDelete = before.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, numBackingIndices)).getIndex(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index cfd2543e9c236..1a63baba882a6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -23,9 +23,9 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexResponse.IndexResult; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamRequestTests; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.block.ClusterBlock; @@ -401,7 +401,7 @@ public void testCloseCurrentWriteIndexForDataStream() { dataStreamsToCreate.add(new Tuple<>(dataStreamName, numBackingIndices)); writeIndices.add(DataStream.getDefaultBackingIndexName(dataStreamName, numBackingIndices)); } - ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dataStreamsToCreate, + ClusterState cs = DataStreamTestHelper.getClusterStateWithDataStreams(dataStreamsToCreate, org.elasticsearch.common.collect.List.of()); ClusterService clusterService = mock(ClusterService.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 079fcf27a2537..5614cbcaa2e72 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -1275,7 +1275,7 @@ public static Metadata randomMetadata() { .put("component_template_" + randomAlphaOfLength(3), ComponentTemplateTests.randomInstance()) .put("index_template_v2_" + randomAlphaOfLength(3), ComposableIndexTemplateTests.randomInstance()); - DataStream randomDataStream = DataStreamTests.randomInstance(); + DataStream randomDataStream = DataStreamTestHelper.randomInstance(); for (Index index : randomDataStream.getIndices()) { md.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java index 387a6f0031117..29858a5b24914 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java @@ -16,17 +16,27 @@ * specific language governing permissions and limitations * under the License. */ - package org.elasticsearch.cluster; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; public final class DataStreamTestHelper { @@ -78,4 +88,64 @@ public static String generateMapping(String timestampFieldName, String type) { " }\n" + " }"; } + + public static List randomIndexInstances() { + int numIndices = ESTestCase.randomIntBetween(0, 128); + List indices = new ArrayList<>(numIndices); + for (int i = 0; i < numIndices; i++) { + indices.add(new Index(randomAlphaOfLength(10).toLowerCase(Locale.ROOT), UUIDs.randomBase64UUID(LuceneTestCase.random()))); + } + return indices; + } + + public static DataStream randomInstance() { + List indices = randomIndexInstances(); + long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random()))); + return new DataStream(dataStreamName, createTimestampField("@timestamp"), indices, generation); + } + + /** + * Constructs {@code ClusterState} with the specified data streams and indices. + * + * @param dataStreams The names of the data streams to create with their respective number of backing indices + * @param indexNames The names of indices to create that do not back any data streams + */ + public static ClusterState getClusterStateWithDataStreams(List> dataStreams, List indexNames) { + Metadata.Builder builder = Metadata.builder(); + + List allIndices = new ArrayList<>(); + for (Tuple dsTuple : dataStreams) { + List backingIndices = new ArrayList<>(); + for (int backingIndexNumber = 1; backingIndexNumber <= dsTuple.v2(); backingIndexNumber++) { + backingIndices.add(createIndexMetadata(getDefaultBackingIndexName(dsTuple.v1(), backingIndexNumber), true)); + } + allIndices.addAll(backingIndices); + + DataStream ds = new DataStream( + dsTuple.v1(), + createTimestampField("@timestamp"), + backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()), + dsTuple.v2() + ); + builder.put(ds); + } + + for (String indexName : indexNames) { + allIndices.add(createIndexMetadata(indexName, false)); + } + + for (IndexMetadata index : allIndices) { + builder.put(index, false); + } + + return ClusterState.builder(new ClusterName("_name")).metadata(builder).build(); + } + + private static IndexMetadata createIndexMetadata(String name, boolean hidden) { + Settings.Builder b = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put("index.hidden", hidden); + + return IndexMetadata.builder(name).settings(b).numberOfShards(1).numberOfReplicas(1).build(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java index 130ec2ce8bd65..b11841cffc6ae 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestCluster.java @@ -20,14 +20,11 @@ package org.elasticsearch.test; import com.carrotsearch.hppc.ObjectArrayList; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -79,7 +76,6 @@ public void beforeTest(Random random, double transportClientRatio) throws IOExce * Wipes any data that a test can leave behind: indices, templates (except exclude templates) and repositories */ public void wipe(Set excludeTemplates) { - wipeAllDataStreams(); wipeIndices("_all"); wipeAllTemplates(excludeTemplates); wipeRepositories(); @@ -135,18 +131,6 @@ public void assertAfterTest() throws IOException { @Override public abstract void close() throws IOException; - /** - * Deletes all data streams from the test cluster. - */ - public void wipeAllDataStreams() { - // Feature flag may not be enabled in all gradle modules that use ESIntegTestCase - if (size() > 0) { - AcknowledgedResponse response = - client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"*"})).actionGet(); - assertAcked(response); - } - } - /** * Deletes the given indices from the tests cluster. If no index name is passed to this method * all indices are removed. diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a52147161767d..0777061518084 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -687,7 +687,9 @@ protected static void wipeAllIndices() throws IOException { protected static void wipeDataStreams() throws IOException { try { - adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + if (hasXPack()) { + adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + } } catch (ResponseException e) { // We hit a version of ES that doesn't have data streams enabled so it's safe to ignore if (e.getResponse().getStatusLine().getStatusCode() != 405 && e.getResponse().getStatusLine().getStatusCode() != 500) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/CreateDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/CreateDataStreamAction.java new file mode 100644 index 0000000000000..bd65c8e82103a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/CreateDataStreamAction.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +public class CreateDataStreamAction extends ActionType { + + public static final CreateDataStreamAction INSTANCE = new CreateDataStreamAction(); + public static final String NAME = "indices:admin/data_stream/create"; + + private CreateDataStreamAction() { + super(NAME, AcknowledgedResponse::new); + } + + public static class Request extends AcknowledgedRequest implements IndicesRequest { + + private final String name; + + public Request(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(name) == false) { + validationException = ValidateActions.addValidationError("name is missing", validationException); + } + return validationException; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return name.equals(request.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String[] indices() { + return new String[]{name}; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamsStatsAction.java new file mode 100644 index 0000000000000..449f7b10dde16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamsStatsAction.java @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastRequest; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.store.StoreStats; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class DataStreamsStatsAction extends ActionType { + + public static final DataStreamsStatsAction INSTANCE = new DataStreamsStatsAction(); + public static final String NAME = "indices:monitor/data_stream/stats"; + + public DataStreamsStatsAction() { + super(NAME, DataStreamsStatsAction.Response::new); + } + + public static class Request extends BroadcastRequest { + public Request() { + super((String[]) null); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + } + + public static class Response extends BroadcastResponse { + private final int dataStreamCount; + private final int backingIndices; + private final ByteSizeValue totalStoreSize; + private final DataStreamStats[] dataStreams; + + public Response(int totalShards, int successfulShards, int failedShards, List shardFailures, + int dataStreamCount, int backingIndices, ByteSizeValue totalStoreSize, DataStreamStats[] dataStreams) { + super(totalShards, successfulShards, failedShards, shardFailures); + this.dataStreamCount = dataStreamCount; + this.backingIndices = backingIndices; + this.totalStoreSize = totalStoreSize; + this.dataStreams = dataStreams; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.dataStreamCount = in.readVInt(); + this.backingIndices = in.readVInt(); + this.totalStoreSize = new ByteSizeValue(in); + this.dataStreams = in.readArray(DataStreamStats::new, DataStreamStats[]::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVInt(dataStreamCount); + out.writeVInt(backingIndices); + totalStoreSize.writeTo(out); + out.writeArray(dataStreams); + } + + @Override + protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { + builder.field("data_stream_count", dataStreamCount); + builder.field("backing_indices", backingIndices); + builder.humanReadableField("total_store_size_bytes", "total_store_size", totalStoreSize); + builder.array("data_streams", (Object[]) dataStreams); + } + + public int getDataStreamCount() { + return dataStreamCount; + } + + public int getBackingIndices() { + return backingIndices; + } + + public ByteSizeValue getTotalStoreSize() { + return totalStoreSize; + } + + public DataStreamStats[] getDataStreams() { + return dataStreams; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Response response = (Response) obj; + return dataStreamCount == response.dataStreamCount && + backingIndices == response.backingIndices && + Objects.equals(totalStoreSize, response.totalStoreSize) && + Arrays.equals(dataStreams, response.dataStreams); + } + + @Override + public int hashCode() { + int result = Objects.hash(dataStreamCount, backingIndices, totalStoreSize); + result = 31 * result + Arrays.hashCode(dataStreams); + return result; + } + + @Override + public String toString() { + return "Response{" + + "dataStreamCount=" + dataStreamCount + + ", backingIndices=" + backingIndices + + ", totalStoreSize=" + totalStoreSize + + ", dataStreams=" + Arrays.toString(dataStreams) + + '}'; + } + } + + public static class DataStreamStats implements ToXContentObject, Writeable { + private final String dataStream; + private final int backingIndices; + private final ByteSizeValue storeSize; + private final long maximumTimestamp; + + public DataStreamStats(String dataStream, int backingIndices, ByteSizeValue storeSize, long maximumTimestamp) { + this.dataStream = dataStream; + this.backingIndices = backingIndices; + this.storeSize = storeSize; + this.maximumTimestamp = maximumTimestamp; + } + + public DataStreamStats(StreamInput in) throws IOException { + this.dataStream = in.readString(); + this.backingIndices = in.readVInt(); + this.storeSize = new ByteSizeValue(in); + this.maximumTimestamp = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStream); + out.writeVInt(backingIndices); + storeSize.writeTo(out); + out.writeVLong(maximumTimestamp); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("data_stream", dataStream); + builder.field("backing_indices", backingIndices); + builder.humanReadableField("store_size_bytes", "store_size", storeSize); + builder.field("maximum_timestamp", maximumTimestamp); + builder.endObject(); + return builder; + } + + public String getDataStream() { + return dataStream; + } + + public int getBackingIndices() { + return backingIndices; + } + + public ByteSizeValue getStoreSize() { + return storeSize; + } + + public long getMaximumTimestamp() { + return maximumTimestamp; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + DataStreamStats that = (DataStreamStats) obj; + return backingIndices == that.backingIndices && + maximumTimestamp == that.maximumTimestamp && + Objects.equals(dataStream, that.dataStream) && + Objects.equals(storeSize, that.storeSize); + } + + @Override + public int hashCode() { + return Objects.hash(dataStream, backingIndices, storeSize, maximumTimestamp); + } + + @Override + public String toString() { + return "DataStreamStats{" + + "dataStream='" + dataStream + '\'' + + ", backingIndices=" + backingIndices + + ", storeSize=" + storeSize + + ", maximumTimestamp=" + maximumTimestamp + + '}'; + } + } + + public static class DataStreamShardStats implements Writeable { + private final ShardRouting shardRouting; + private final StoreStats storeStats; + private final long maxTimestamp; + + public DataStreamShardStats(ShardRouting shardRouting, StoreStats storeStats, long maxTimestamp) { + this.shardRouting = shardRouting; + this.storeStats = storeStats; + this.maxTimestamp = maxTimestamp; + } + + public DataStreamShardStats(StreamInput in) throws IOException { + this.shardRouting = new ShardRouting(in); + this.storeStats = new StoreStats(in); + this.maxTimestamp = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + shardRouting.writeTo(out); + storeStats.writeTo(out); + out.writeVLong(maxTimestamp); + } + + public ShardRouting getShardRouting() { + return shardRouting; + } + + public StoreStats getStoreStats() { + return storeStats; + } + + public long getMaxTimestamp() { + return maxTimestamp; + } + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java new file mode 100644 index 0000000000000..1cc03022d6e7f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DeleteDataStreamAction.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.CollectionUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class DeleteDataStreamAction extends ActionType { + + public static final DeleteDataStreamAction INSTANCE = new DeleteDataStreamAction(); + public static final String NAME = "indices:admin/data_stream/delete"; + + private DeleteDataStreamAction() { + super(NAME, AcknowledgedResponse::new); + } + + public static class Request extends MasterNodeRequest implements IndicesRequest.Replaceable { + + private String[] names; + + public Request(String[] names) { + this.names = Objects.requireNonNull(names); + } + + public String[] getNames() { + return names; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (CollectionUtils.isEmpty(names)) { + validationException = addValidationError("no data stream(s) specified", validationException); + } + return validationException; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.names = in.readStringArray(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringArray(names); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Arrays.equals(names, request.names); + } + + @Override + public int hashCode() { + return Arrays.hashCode(names); + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and + // a data stream's backing indices are retrieved from its metadata + return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java similarity index 62% rename from server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java index 16696eb71535d..0bac97981725b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/GetDataStreamAction.java @@ -1,64 +1,30 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.action.admin.indices.datastream; +package org.elasticsearch.xpack.core.action; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.AbstractDiffable; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.Index; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; public class GetDataStreamAction extends ActionType { @@ -77,6 +43,10 @@ public Request(String[] names) { this.names = names; } + public String[] getNames() { + return names; + } + @Override public ActionRequestValidationException validate() { return null; @@ -262,62 +232,4 @@ public int hashCode() { } } - public static class TransportAction extends TransportMasterNodeReadAction { - - private static final Logger logger = LogManager.getLogger(TransportAction.class); - - @Inject - public TransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, - ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); - } - - @Override - protected String executor() { - return ThreadPool.Names.SAME; - } - - @Override - protected Response read(StreamInput in) throws IOException { - return new Response(in); - } - - @Override - protected void masterOperation(Request request, ClusterState state, - ActionListener listener) throws Exception { - List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); - List dataStreamInfos = new ArrayList<>(dataStreams.size()); - for (DataStream dataStream : dataStreams) { - String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); - String ilmPolicyName = null; - if (indexTemplate != null) { - Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); - ilmPolicyName = settings.get("index.lifecycle.name"); - } else { - logger.warn("couldn't find any matching template for data stream [{}]. has it been restored (and possibly renamed)" + - "from a snapshot?", dataStream.getName()); - } - ClusterStateHealth streamHealth = new ClusterStateHealth(state, - dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new)); - dataStreamInfos.add(new Response.DataStreamInfo(dataStream, streamHealth.getStatus(), indexTemplate, ilmPolicyName)); - } - listener.onResponse(new Response(dataStreamInfos)); - } - - static List getDataStreams(ClusterState clusterState, IndexNameExpressionResolver iner, Request request) { - List results = iner.dataStreamNames(clusterState, request.indicesOptions(), request.names); - Map dataStreams = clusterState.metadata().dataStreams(); - - return results.stream() - .map(dataStreams::get) - .sorted(Comparator.comparing(DataStream::getName)) - .collect(Collectors.toList()); - } - - @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); - } - } - } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java index 5cbe999d5e964..cb65ae72956e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java @@ -14,9 +14,9 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsAction; diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 9a0ba9bd745fe..60448e4a968f0 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -9,9 +9,6 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; @@ -34,6 +31,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; @@ -54,6 +52,9 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import org.junit.After; @@ -70,6 +71,7 @@ import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; @@ -82,6 +84,7 @@ import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; +@ESIntegTestCase.ClusterScope(transportClientRatio = 0) public class DataStreamIT extends ESIntegTestCase { @Override @@ -90,7 +93,13 @@ protected Collection> nodePlugins() { } @After - public void deleteAllComposableTemplates() { + public void cleanup() { + AcknowledgedResponse response = client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(new String[] { "*" }) + ).actionGet(); + assertAcked(response); + DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[] { "*" }); client().execute(DeleteDataStreamAction.INSTANCE, deleteDSRequest).actionGet(); DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); @@ -100,14 +109,15 @@ public void deleteAllComposableTemplates() { public void testBasicScenario() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); putComposableIndexTemplate("id2", List.of("metrics-bar*")); createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); - GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2)); DataStream firstDataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream(); @@ -174,8 +184,8 @@ public void testBasicScenario() throws Exception { verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2); DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { "metrics-*" }); - client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); - getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet(); + getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); expectThrows( @@ -212,7 +222,7 @@ public void testOtherWriteOps() throws Exception { putComposableIndexTemplate("id", List.of("metrics-foobar*")); String dataStreamName = "metrics-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); { BulkRequest bulkRequest = new BulkRequest().add( @@ -305,7 +315,8 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); - GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getTimeStampField().getName(), equalTo("@timestamp")); @@ -338,8 +349,8 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except verifyDocs(dataStreamName, numDocs + numDocs2, 1, 2); DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); - client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); - getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + client().execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest).actionGet(); + getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); expectThrows( @@ -394,7 +405,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { putComposableIndexTemplate("id", List.of("logs-*")); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(request).actionGet(); + client().execute(CreateDataStreamAction.INSTANCE, request).actionGet(); verifyResolvability( dataStreamName, @@ -449,7 +460,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, client().admin().indices().prepareShardStores(dataStreamName), false); request = new CreateDataStreamAction.Request("logs-barbaz"); - client().admin().indices().createDataStream(request).actionGet(); + client().execute(CreateDataStreamAction.INSTANCE, request).actionGet(); verifyResolvability( "logs-barbaz", client().prepareIndex("logs-barbaz", "_doc") @@ -509,9 +520,9 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio putComposableIndexTemplate("id", List.of("metrics-foobar*")); String dataStreamName = "metrics-foobar-baz"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName + "-eggplant"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); DeleteComposableIndexTemplateAction.Request req = new DeleteComposableIndexTemplateAction.Request("id"); Exception e = expectThrows(Exception.class, () -> client().execute(DeleteComposableIndexTemplateAction.INSTANCE, req).get()); @@ -542,7 +553,7 @@ public void testAliasActionsFailOnDataStreams() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); IndicesAliasesRequest.AliasActions addAction = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD) .index(dataStreamName) @@ -557,7 +568,7 @@ public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); IndicesAliasesRequest.AliasActions addAction = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD) @@ -594,9 +605,10 @@ public void testTimestampFieldCustomAttributes() throws Exception { putComposableIndexTemplate("id1", mapping, List.of("logs-foo*"), null); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "logs-foobar" }); - GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getTimeStampField().getName(), equalTo("@timestamp")); @@ -607,7 +619,7 @@ public void testTimestampFieldCustomAttributes() throws Exception { public void testUpdateMappingViaDataStream() throws Exception { putComposableIndexTemplate("id1", List.of("logs-*")); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); - client().admin().indices().createDataStream(createDataStreamRequest).actionGet(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet(); String backingIndex1 = DataStream.getDefaultBackingIndexName("logs-foobar", 1); String backingIndex2 = DataStream.getDefaultBackingIndexName("logs-foobar", 2); @@ -649,7 +661,7 @@ public void testUpdateMappingViaDataStream() throws Exception { public void testUpdateIndexSettingsViaDataStream() throws Exception { putComposableIndexTemplate("id1", List.of("logs-*")); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); - client().admin().indices().createDataStream(createDataStreamRequest).actionGet(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet(); String backingIndex1 = DataStream.getDefaultBackingIndexName("logs-foobar", 1); String backingIndex2 = DataStream.getDefaultBackingIndexName("logs-foobar", 2); @@ -750,11 +762,11 @@ public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Excepti public void testSearchAllResolvesDataStreams() throws Exception { putComposableIndexTemplate("id1", List.of("metrics-foo*")); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("metrics-foo"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); putComposableIndexTemplate("id2", List.of("metrics-bar*")); createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); int numDocsBar = randomIntBetween(2, 16); indexDocs("metrics-bar", numDocsBar); @@ -780,10 +792,10 @@ public void testGetDataStream() throws Exception { int numDocsFoo = randomIntBetween(2, 16); indexDocs("metrics-foo", numDocsFoo); - GetDataStreamAction.Response response = client().admin() - .indices() - .getDataStreams(new GetDataStreamAction.Request(new String[] { "metrics-foo" })) - .actionGet(); + GetDataStreamAction.Response response = client().execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "metrics-foo" }) + ).actionGet(); assertThat(response.getDataStreams().size(), is(1)); GetDataStreamAction.Response.DataStreamInfo metricsFooDataStream = response.getDataStreams().get(0); assertThat(metricsFooDataStream.getDataStream().getName(), is("metrics-foo")); @@ -808,7 +820,7 @@ public void testNoTimestampInDocument() throws Exception { putComposableIndexTemplate("id", List.of("logs-foobar*")); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON); Exception e = expectThrows(MapperParsingException.class, () -> client().index(indexRequest).actionGet()); @@ -819,7 +831,7 @@ public void testMultipleTimestampValuesInDocument() throws Exception { putComposableIndexTemplate("id", List.of("logs-foobar*")); String dataStreamName = "logs-foobar"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); - client().admin().indices().createDataStream(createDataStreamRequest).get(); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create") .source("{\"@timestamp\": [\"2020-12-12\",\"2022-12-12\"]}", XContentType.JSON); @@ -869,7 +881,8 @@ public void testMixedAutoCreate() throws Exception { assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); - GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + GetDataStreamAction.Response getDataStreamsResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4)); getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); assertThat(getDataStreamsResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); @@ -905,7 +918,8 @@ public void testAutoCreateV1TemplateNoDataStream() { assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); - GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + GetDataStreamAction.Response getDataStreamsResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); assertThat(getDataStreamsResponse.getDataStreams(), hasSize(0)); GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-foobar")).actionGet(); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index f665e05d95308..f01c774188579 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -11,9 +11,10 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -30,6 +31,7 @@ import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import java.nio.file.Path; @@ -44,6 +46,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +@ESIntegTestCase.ClusterScope(transportClientRatio = 0) public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { private static final String DS_BACKING_INDEX_NAME = DataStream.getDefaultBackingIndexName("ds", 1); @@ -69,11 +72,11 @@ public void setup() throws Exception { DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds")); CreateDataStreamAction.Request request = new CreateDataStreamAction.Request("ds"); - AcknowledgedResponse response = client.admin().indices().createDataStream(request).get(); + AcknowledgedResponse response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); assertTrue(response.isAcknowledged()); request = new CreateDataStreamAction.Request("other-ds"); - response = client.admin().indices().createDataStream(request).get(); + response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); assertTrue(response.isAcknowledged()); IndexResponse indexResponse = client.prepareIndex("ds", "_doc") @@ -84,6 +87,15 @@ public void setup() throws Exception { id = indexResponse.getId(); } + @After + public void cleanup() { + AcknowledgedResponse response = client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(new String[] { "*" }) + ).actionGet(); + assertAcked(response); + } + public void testSnapshotAndRestore() throws Exception { CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() @@ -102,7 +114,9 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), snap.get(0).indices()); assertTrue( - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get().isAcknowledged() + client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })) + .get() + .isAcknowledged() ); RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -119,10 +133,10 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(1, hits.length); assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); - GetDataStreamAction.Response ds = client.admin() - .indices() - .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds" })) - .get(); + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "ds" }) + ).get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); @@ -145,7 +159,7 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(1, snap.size()); assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), snap.get(0).indices()); - assertAcked(client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "*" })).get()); + assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -162,15 +176,15 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(1, hits.length); assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); - GetDataStreamAction.Response ds = client.admin() - .indices() - .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds" })) - .get(); + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "ds" }) + ).get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertAcked(client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get()); + assertAcked(client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).get()); } public void testRename() throws Exception { @@ -199,10 +213,10 @@ public void testRename() throws Exception { .setRenameReplacement("ds2") .get(); - GetDataStreamAction.Response ds = client.admin() - .indices() - .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds2" })) - .get(); + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "ds2" }) + ).get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); @@ -228,7 +242,7 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { ); // delete data stream - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).actionGet(); + client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })).actionGet(); // restore data stream attempting to rename the backing index RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -243,7 +257,7 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK)); GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" }); - GetDataStreamAction.Response response = client.admin().indices().getDataStreams(getDSRequest).actionGet(); + GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet(); assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME)); } @@ -278,7 +292,7 @@ public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() { // assert "ds" was restored as "test-ds" and the backing index has a valid name GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(new String[] { "test-ds" }); - GetDataStreamAction.Response response = client.admin().indices().getDataStreams(getRenamedDS).actionGet(); + GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getRenamedDS).actionGet(); assertThat( response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DataStream.getDefaultBackingIndexName("test-ds", 1L)) @@ -286,7 +300,7 @@ public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() { // data stream "ds" should still exist in the system GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" }); - response = client.admin().indices().getDataStreams(getDSRequest).actionGet(); + response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet(); assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME)); } @@ -313,10 +327,10 @@ public void testWildcards() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); - GetDataStreamAction.Response ds = client.admin() - .indices() - .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds2" })) - .get(); + GetDataStreamAction.Response ds = client.execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(new String[] { "ds2" }) + ).get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); @@ -327,7 +341,7 @@ public void testWildcards() throws Exception { ); } - public void testDataStreamNotStoredWhenIndexRequested() throws Exception { + public void testDataStreamNotStoredWhenIndexRequested() { CreateSnapshotResponse createSnapshotResponse = client.admin() .cluster() .prepareCreateSnapshot(REPO, "snap2") @@ -357,7 +371,9 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { assertEquals(RestStatus.OK, status); assertTrue( - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get().isAcknowledged() + client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" })) + .get() + .isAcknowledged() ); RestoreSnapshotResponse restoreSnapshotResponse = client.admin() @@ -370,7 +386,7 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "ds" }); - expectThrows(ResourceNotFoundException.class, () -> client.admin().indices().getDataStreams(getRequest).actionGet()); + expectThrows(ResourceNotFoundException.class, () -> client.execute(GetDataStreamAction.INSTANCE, getRequest).actionGet()); } public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionException, InterruptedException { @@ -385,7 +401,9 @@ public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionExcepti assertThat(createSnapshotResponse.getSnapshotInfo().state(), Matchers.is(SnapshotState.SUCCESS)); assertThat( - client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "*" })).get().isAcknowledged(), + client().execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })) + .get() + .isAcknowledged(), is(true) ); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java index fa5d47040e264..34bfd7d7db51f 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.collect.List; @@ -24,16 +26,19 @@ import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; +import org.junit.After; import java.util.Collection; import java.util.Collections; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; // The tests in here do a lot of state updates and other writes to disk and are slowed down too much by WindowsFS @LuceneTestCase.SuppressFileSystems(value = "WindowsFS") +@ESIntegTestCase.ClusterScope(transportClientRatio = 0) public class ShardClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { @Override @@ -41,6 +46,15 @@ protected Collection> nodePlugins() { return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class); } + @After + public void cleanup() { + AcknowledgedResponse response = client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(new String[] { "*" }) + ).actionGet(); + assertAcked(response); + } + public void testDeleteDataStreamDuringSnapshot() throws Exception { Client client = client(); @@ -83,7 +97,7 @@ public void testDeleteDataStreamDuringSnapshot() throws Exception { // non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed try { logger.info("--> delete index while non-partial snapshot is running"); - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet(); + client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet(); fail("Expected deleting index to fail during snapshot"); } catch (SnapshotInProgressException e) { assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: [" + dataStream)); diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java index 0c4ee6a0a8f2a..2093c14b2001c 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java @@ -3,23 +3,45 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.xpack.datastreams; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.inject.Module; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.xpack.datastreams.action.DataStreamsStatsTransportAction; +import org.elasticsearch.xpack.datastreams.rest.RestCreateDataStreamAction; +import org.elasticsearch.xpack.datastreams.rest.RestDataStreamsStatsAction; +import org.elasticsearch.xpack.datastreams.rest.RestDeleteDataStreamAction; +import org.elasticsearch.xpack.datastreams.rest.RestGetDataStreamsAction; import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.datastreams.action.CreateDataStreamTransportAction; +import org.elasticsearch.xpack.datastreams.action.DeleteDataStreamTransportAction; +import org.elasticsearch.xpack.datastreams.action.GetDataStreamsTransportAction; import org.elasticsearch.xpack.datastreams.mapper.DataStreamTimestampFieldMapper; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class DataStreamsPlugin extends Plugin implements ActionPlugin, MapperPlugin { @@ -34,6 +56,33 @@ public Map getMetadataMappers() { return Collections.singletonMap(DataStreamTimestampFieldMapper.NAME, new DataStreamTimestampFieldMapper.TypeParser()); } + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(CreateDataStreamAction.INSTANCE, CreateDataStreamTransportAction.class), + new ActionHandler<>(DeleteDataStreamAction.INSTANCE, DeleteDataStreamTransportAction.class), + new ActionHandler<>(GetDataStreamAction.INSTANCE, GetDataStreamsTransportAction.class), + new ActionHandler<>(DataStreamsStatsAction.INSTANCE, DataStreamsStatsTransportAction.class) + ); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + RestHandler createDsAction = new RestCreateDataStreamAction(); + RestHandler deleteDsAction = new RestDeleteDataStreamAction(); + RestHandler getDsAction = new RestGetDataStreamsAction(); + RestHandler dsStatsAction = new RestDataStreamsStatsAction(); + return Arrays.asList(createDsAction, deleteDsAction, getDsAction, dsStatsAction); + } + public Collection createGuiceModules() { List modules = new ArrayList<>(); diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java new file mode 100644 index 0000000000000..33acf29160a57 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/CreateDataStreamTransportAction.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datastreams.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; + +import java.io.IOException; + +public class CreateDataStreamTransportAction extends TransportMasterNodeAction { + + private final MetadataCreateDataStreamService metadataCreateDataStreamService; + + @Inject + public CreateDataStreamTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataCreateDataStreamService metadataCreateDataStreamService + ) { + super( + CreateDataStreamAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + CreateDataStreamAction.Request::new, + indexNameExpressionResolver + ); + this.metadataCreateDataStreamService = metadataCreateDataStreamService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse read(StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + protected void masterOperation( + CreateDataStreamAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest updateRequest = + new MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest( + request.getName(), + request.masterNodeTimeout(), + request.timeout() + ); + metadataCreateDataStreamService.createDataStream(updateRequest, listener); + } + + @Override + protected ClusterBlockException checkBlock(CreateDataStreamAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java new file mode 100644 index 0000000000000..3d5ec48f30394 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DataStreamsStatsTransportAction.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.PointValues; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexAbstractionResolver; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardsIterator; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardNotFoundException; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.stream.Stream; + +public class DataStreamsStatsTransportAction extends TransportBroadcastByNodeAction< + DataStreamsStatsAction.Request, + DataStreamsStatsAction.Response, + DataStreamsStatsAction.DataStreamShardStats> { + + private final ClusterService clusterService; + private final IndicesService indicesService; + private final IndexAbstractionResolver indexAbstractionResolver; + + @Inject + public DataStreamsStatsTransportAction( + ClusterService clusterService, + TransportService transportService, + IndicesService indicesService, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + DataStreamsStatsAction.NAME, + clusterService, + transportService, + actionFilters, + indexNameExpressionResolver, + DataStreamsStatsAction.Request::new, + ThreadPool.Names.MANAGEMENT + ); + this.clusterService = clusterService; + this.indicesService = indicesService; + this.indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + } + + @Override + protected DataStreamsStatsAction.Request readRequestFrom(StreamInput in) throws IOException { + return new DataStreamsStatsAction.Request(in); + } + + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState state, DataStreamsStatsAction.Request request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + @Override + protected ClusterBlockException checkRequestBlock( + ClusterState state, + DataStreamsStatsAction.Request request, + String[] concreteIndices + ) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); + } + + @Override + protected ShardsIterator shards(ClusterState clusterState, DataStreamsStatsAction.Request request, String[] concreteIndices) { + String[] requestIndices = request.indices(); + if (requestIndices == null || requestIndices.length == 0) { + requestIndices = new String[] { "*" }; + } + List abstractionNames = indexAbstractionResolver.resolveIndexAbstractions( + requestIndices, + request.indicesOptions(), + clusterState.getMetadata(), + true + ); // Always include data streams for data streams stats api + SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); + + String[] concreteDatastreamIndices = abstractionNames.stream().flatMap(abstractionName -> { + IndexAbstraction indexAbstraction = indicesLookup.get(abstractionName); + assert indexAbstraction != null; + if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { + IndexAbstraction.DataStream dataStream = (IndexAbstraction.DataStream) indexAbstraction; + List indices = dataStream.getIndices(); + return indices.stream().map(idx -> idx.getIndex().getName()); + } else { + return Stream.empty(); + } + }).toArray(String[]::new); + return clusterState.getRoutingTable().allShards(concreteDatastreamIndices); + } + + @Override + protected DataStreamsStatsAction.DataStreamShardStats shardOperation(DataStreamsStatsAction.Request request, ShardRouting shardRouting) + throws IOException { + IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); + IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); + // if we don't have the routing entry yet, we need it stats wise, we treat it as if the shard is not ready yet + if (indexShard.routingEntry() == null) { + throw new ShardNotFoundException(indexShard.shardId()); + } + StoreStats storeStats = indexShard.storeStats(); + IndexAbstraction indexAbstraction = clusterService.state().getMetadata().getIndicesLookup().get(shardRouting.getIndexName()); + assert indexAbstraction != null; + IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); + assert dataStream != null; + long maxTimestamp = 0L; + try (Engine.Searcher searcher = indexShard.acquireSearcher("data_stream_stats")) { + IndexReader indexReader = searcher.getIndexReader(); + String fieldName = dataStream.getDataStream().getTimeStampField().getName(); + byte[] maxPackedValue = PointValues.getMaxPackedValue(indexReader, fieldName); + if (maxPackedValue != null) { + maxTimestamp = LongPoint.decodeDimension(maxPackedValue, 0); + } + } + return new DataStreamsStatsAction.DataStreamShardStats(indexShard.routingEntry(), storeStats, maxTimestamp); + } + + @Override + protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInput in) throws IOException { + return new DataStreamsStatsAction.DataStreamShardStats(in); + } + + @Override + protected DataStreamsStatsAction.Response newResponse( + DataStreamsStatsAction.Request request, + int totalShards, + int successfulShards, + int failedShards, + List dataStreamShardStats, + List shardFailures, + ClusterState clusterState + ) { + Map aggregatedDataStreamsStats = new HashMap<>(); + Set allBackingIndices = new HashSet<>(); + long totalStoreSizeBytes = 0L; + + SortedMap indicesLookup = clusterState.getMetadata().getIndicesLookup(); + for (DataStreamsStatsAction.DataStreamShardStats shardStat : dataStreamShardStats) { + String indexName = shardStat.getShardRouting().getIndexName(); + IndexAbstraction indexAbstraction = indicesLookup.get(indexName); + IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream(); + assert dataStream != null; + + // Aggregate global stats + totalStoreSizeBytes += shardStat.getStoreStats().sizeInBytes(); + allBackingIndices.add(indexName); + + // Aggregate data stream stats + AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats()); + stats.storageBytes += shardStat.getStoreStats().sizeInBytes(); + stats.maxTimestamp = Math.max(stats.maxTimestamp, shardStat.getMaxTimestamp()); + stats.backingIndices.add(indexName); + } + + DataStreamsStatsAction.DataStreamStats[] dataStreamStats = aggregatedDataStreamsStats.entrySet() + .stream() + .map( + entry -> new DataStreamsStatsAction.DataStreamStats( + entry.getKey(), + entry.getValue().backingIndices.size(), + new ByteSizeValue(entry.getValue().storageBytes), + entry.getValue().maxTimestamp + ) + ) + .toArray(DataStreamsStatsAction.DataStreamStats[]::new); + + return new DataStreamsStatsAction.Response( + totalShards, + successfulShards, + failedShards, + shardFailures, + aggregatedDataStreamsStats.size(), + allBackingIndices.size(), + new ByteSizeValue(totalStoreSizeBytes), + dataStreamStats + ); + } + + private static class AggregatedStats { + Set backingIndices = new HashSet<>(); + long storageBytes = 0L; + long maxTimestamp = 0L; + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java new file mode 100644 index 0000000000000..abdef5bd3ddf7 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportAction.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Priority; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.snapshots.SnapshotInProgressException; +import org.elasticsearch.snapshots.SnapshotsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +public class DeleteDataStreamTransportAction extends TransportMasterNodeAction { + + private static final Logger LOGGER = LogManager.getLogger(DeleteDataStreamTransportAction.class); + + private final MetadataDeleteIndexService deleteIndexService; + + @Inject + public DeleteDataStreamTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataDeleteIndexService deleteIndexService + ) { + super( + DeleteDataStreamAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + DeleteDataStreamAction.Request::new, + indexNameExpressionResolver + ); + this.deleteIndexService = deleteIndexService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse read(StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + protected void masterOperation( + DeleteDataStreamAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + clusterService.submitStateUpdateTask( + "remove-data-stream [" + Strings.arrayToCommaDelimitedString(request.getNames()) + "]", + new ClusterStateUpdateTask(Priority.HIGH) { + + @Override + public TimeValue timeout() { + return request.masterNodeTimeout(); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public ClusterState execute(ClusterState currentState) { + return removeDataStream(deleteIndexService, currentState, request); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(new AcknowledgedResponse(true)); + } + } + ); + } + + static ClusterState removeDataStream( + MetadataDeleteIndexService deleteIndexService, + ClusterState currentState, + DeleteDataStreamAction.Request request + ) { + Set dataStreams = new HashSet<>(); + Set snapshottingDataStreams = new HashSet<>(); + for (String name : request.getNames()) { + for (String dataStreamName : currentState.metadata().dataStreams().keySet()) { + if (Regex.simpleMatch(name, dataStreamName)) { + dataStreams.add(dataStreamName); + } + } + + snapshottingDataStreams.addAll(SnapshotsService.snapshottingDataStreams(currentState, dataStreams)); + } + + if (snapshottingDataStreams.isEmpty() == false) { + throw new SnapshotInProgressException( + "Cannot delete data streams that are being snapshotted: " + + snapshottingDataStreams + + ". Try again after snapshot finishes or cancel the currently running snapshot." + ); + } + + Set backingIndicesToRemove = new HashSet<>(); + for (String dataStreamName : dataStreams) { + DataStream dataStream = currentState.metadata().dataStreams().get(dataStreamName); + assert dataStream != null; + backingIndicesToRemove.addAll(dataStream.getIndices()); + } + + // first delete the data streams and then the indices: + // (this to avoid data stream validation from failing when deleting an index that is part of a data stream + // without updating the data stream) + // TODO: change order when delete index api also updates the data stream the index to be removed is member of + Metadata.Builder metadata = Metadata.builder(currentState.metadata()); + for (String ds : dataStreams) { + LOGGER.info("removing data stream [{}]", ds); + metadata.removeDataStream(ds); + } + currentState = ClusterState.builder(currentState).metadata(metadata).build(); + return deleteIndexService.deleteIndices(currentState, backingIndicesToRemove); + } + + @Override + protected ClusterBlockException checkBlock(DeleteDataStreamAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java new file mode 100644 index 0000000000000..8e09acd8c675e --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportAction.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class GetDataStreamsTransportAction extends TransportMasterNodeReadAction< + GetDataStreamAction.Request, + GetDataStreamAction.Response> { + + private static final Logger LOGGER = LogManager.getLogger(GetDataStreamsTransportAction.class); + + @Inject + public GetDataStreamsTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + GetDataStreamAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamAction.Request::new, + indexNameExpressionResolver + ); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected GetDataStreamAction.Response read(StreamInput in) throws IOException { + return new GetDataStreamAction.Response(in); + } + + @Override + protected void masterOperation( + GetDataStreamAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); + List dataStreamInfos = new ArrayList<>(dataStreams.size()); + for (DataStream dataStream : dataStreams) { + String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); + String ilmPolicyName = null; + if (indexTemplate != null) { + Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); + ilmPolicyName = settings.get("index.lifecycle.name"); + } else { + LOGGER.warn( + "couldn't find any matching template for data stream [{}]. has it been restored (and possibly renamed)" + + "from a snapshot?", + dataStream.getName() + ); + } + ClusterStateHealth streamHealth = new ClusterStateHealth( + state, + dataStream.getIndices().stream().map(Index::getName).toArray(String[]::new) + ); + dataStreamInfos.add( + new GetDataStreamAction.Response.DataStreamInfo(dataStream, streamHealth.getStatus(), indexTemplate, ilmPolicyName) + ); + } + listener.onResponse(new GetDataStreamAction.Response(dataStreamInfos)); + } + + static List getDataStreams( + ClusterState clusterState, + IndexNameExpressionResolver iner, + GetDataStreamAction.Request request + ) { + List results = iner.dataStreamNames(clusterState, request.indicesOptions(), request.getNames()); + Map dataStreams = clusterState.metadata().dataStreams(); + + return results.stream().map(dataStreams::get).sorted(Comparator.comparing(DataStream::getName)).collect(Collectors.toList()); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestCreateDataStreamAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestCreateDataStreamAction.java new file mode 100644 index 0000000000000..4c00c6b7aaf22 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestCreateDataStreamAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datastreams.rest; + +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class RestCreateDataStreamAction extends BaseRestHandler { + + @Override + public String getName() { + return "create_data_stream_action"; + } + + @Override + public List routes() { + return Collections.singletonList(new Route(RestRequest.Method.PUT, "/_data_stream/{name}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + CreateDataStreamAction.Request putDataStreamRequest = new CreateDataStreamAction.Request(request.param("name")); + return channel -> client.execute(CreateDataStreamAction.INSTANCE, putDataStreamRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDataStreamsStatsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java similarity index 62% rename from server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDataStreamsStatsAction.java rename to x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java index a9fc8816d4870..3e3f6f2f8194a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDataStreamsStatsAction.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDataStreamsStatsAction.java @@ -1,31 +1,17 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ +package org.elasticsearch.xpack.datastreams.rest; -package org.elasticsearch.rest.action.admin.indices; - -import org.elasticsearch.action.admin.indices.datastream.DataStreamsStatsAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; import java.io.IOException; import java.util.Arrays; @@ -49,10 +35,11 @@ public List routes() { protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { DataStreamsStatsAction.Request dataStreamsStatsRequest = new DataStreamsStatsAction.Request(); boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true); - IndicesOptions defaultIndicesOption = forbidClosedIndices ? dataStreamsStatsRequest.indicesOptions() + IndicesOptions defaultIndicesOption = forbidClosedIndices + ? dataStreamsStatsRequest.indicesOptions() : IndicesOptions.strictExpandOpen(); - assert dataStreamsStatsRequest.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() : "DataStreamStats default " + - "indices options changed"; + assert dataStreamsStatsRequest.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() : "DataStreamStats default " + + "indices options changed"; dataStreamsStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, defaultIndicesOption)); dataStreamsStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("name"))); return channel -> client.execute(DataStreamsStatsAction.INSTANCE, dataStreamsStatsRequest, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java new file mode 100644 index 0000000000000..6282686f3f191 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestDeleteDataStreamAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datastreams.rest; + +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class RestDeleteDataStreamAction extends BaseRestHandler { + + @Override + public String getName() { + return "delete_data_stream_action"; + } + + @Override + public List routes() { + return Collections.singletonList(new Route(RestRequest.Method.DELETE, "/_data_stream/{name}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request( + Strings.splitStringByCommaToArray(request.param("name")) + ); + return channel -> client.execute(DeleteDataStreamAction.INSTANCE, deleteDataStreamRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java new file mode 100644 index 0000000000000..682da19424be9 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/rest/RestGetDataStreamsAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.datastreams.rest; + +import org.elasticsearch.xpack.core.action.GetDataStreamAction; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class RestGetDataStreamsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_data_streams_action"; + } + + @Override + public List routes() { + return Arrays.asList(new Route(RestRequest.Method.GET, "/_data_stream"), new Route(RestRequest.Method.GET, "/_data_stream/{name}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request( + Strings.splitStringByCommaToArray(request.param("name")) + ); + return channel -> client.execute(GetDataStreamAction.INSTANCE, getDataStreamsRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/CreateDataStreamRequestTests.java similarity index 56% rename from server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java rename to x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/CreateDataStreamRequestTests.java index 863c2f3245e06..8589076239bfc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamRequestTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/CreateDataStreamRequestTests.java @@ -1,25 +1,12 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.action.admin.indices.datastream; +package org.elasticsearch.xpack.core.action; import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction.Request; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction.Request; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DataStreamsStatsResponseTests.java similarity index 59% rename from server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java rename to x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DataStreamsStatsResponseTests.java index c64a342e6050d..93f35ad0e4495 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DataStreamsStatsResponseTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DataStreamsStatsResponseTests.java @@ -1,23 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ - -package org.elasticsearch.action.admin.indices.datastream; +package org.elasticsearch.xpack.core.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.DefaultShardOperationFailedException; @@ -53,20 +39,33 @@ public static DataStreamsStatsAction.Response randomStatsResponse() { long storeSize = randomLongBetween(250, 1000000000); totalStoreSize += storeSize; long maximumTimestamp = randomRecentTimestamp(); - dataStreamStats.add(new DataStreamsStatsAction.DataStreamStats(dataStreamName, backingIndices, - new ByteSizeValue(storeSize), maximumTimestamp)); + dataStreamStats.add( + new DataStreamsStatsAction.DataStreamStats(dataStreamName, backingIndices, new ByteSizeValue(storeSize), maximumTimestamp) + ); } int totalShards = randomIntBetween(backingIndicesTotal, backingIndicesTotal * 3); int successfulShards = randomInt(totalShards); int failedShards = totalShards - successfulShards; List exceptions = new ArrayList<>(); for (int i = 0; i < failedShards; i++) { - exceptions.add(new DefaultShardOperationFailedException(randomAlphaOfLength(8).toLowerCase(Locale.getDefault()), - randomInt(totalShards), new ElasticsearchException("boom"))); + exceptions.add( + new DefaultShardOperationFailedException( + randomAlphaOfLength(8).toLowerCase(Locale.getDefault()), + randomInt(totalShards), + new ElasticsearchException("boom") + ) + ); } - return new DataStreamsStatsAction.Response(totalShards, successfulShards, failedShards, exceptions, - dataStreamCount, backingIndicesTotal, new ByteSizeValue(totalStoreSize), - dataStreamStats.toArray(new DataStreamsStatsAction.DataStreamStats[0])); + return new DataStreamsStatsAction.Response( + totalShards, + successfulShards, + failedShards, + exceptions, + dataStreamCount, + backingIndicesTotal, + new ByteSizeValue(totalStoreSize), + dataStreamStats.toArray(new DataStreamsStatsAction.DataStreamStats[0]) + ); } private static long randomRecentTimestamp() { diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DeleteDataStreamRequestTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DeleteDataStreamRequestTests.java new file mode 100644 index 0000000000000..163d1f267166e --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/DeleteDataStreamRequestTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction.Request; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } + + @Override + protected Request createTestInstance() { + return new Request(randomArray(1, 3, String[]::new, () -> randomAlphaOfLength(6))); + } + + public void testValidateRequest() { + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "my-data-stream" }); + ActionRequestValidationException e = req.validate(); + assertNull(e); + } + + public void testValidateRequestWithoutName() { + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[0]); + ActionRequestValidationException e = req.validate(); + assertNotNull(e); + assertThat(e.validationErrors().size(), equalTo(1)); + assertThat(e.validationErrors().get(0), containsString("no data stream(s) specified")); + } + +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsRequestTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsRequestTests.java new file mode 100644 index 0000000000000..0819b9304b473 --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsRequestTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.action.GetDataStreamAction.Request; + +public class GetDataStreamsRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } + + @Override + protected Request createTestInstance() { + final String[] searchParameter; + switch (randomIntBetween(1, 4)) { + case 1: + searchParameter = generateRandomStringArray(3, 8, false, false); + break; + case 2: + String[] parameters = generateRandomStringArray(3, 8, false, false); + for (int k = 0; k < parameters.length; k++) { + parameters[k] = parameters[k] + "*"; + } + searchParameter = parameters; + break; + case 3: + searchParameter = new String[] { "*" }; + break; + default: + searchParameter = null; + break; + } + return new Request(searchParameter); + } + +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsResponseTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsResponseTests.java new file mode 100644 index 0000000000000..5f09eddfd44b9 --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/core/action/GetDataStreamsResponseTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.cluster.DataStreamTestHelper; +import org.elasticsearch.xpack.core.action.GetDataStreamAction.Response; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return Response::new; + } + + @Override + protected Response createTestInstance() { + int numDataStreams = randomIntBetween(0, 8); + List dataStreams = new ArrayList<>(); + for (int i = 0; i < numDataStreams; i++) { + dataStreams.add( + new Response.DataStreamInfo( + DataStreamTestHelper.randomInstance(), + ClusterHealthStatus.GREEN, + randomAlphaOfLengthBetween(2, 10), + randomAlphaOfLengthBetween(2, 10) + ) + ); + } + return new Response(dataStreams); + } +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index 7570df6c3a48c..5ce823280548d 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -7,9 +7,6 @@ package org.elasticsearch.xpack.datastreams; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.DataStreamsStatsAction; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; @@ -22,6 +19,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.DataStreamsStatsAction; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; import org.junit.After; import java.util.Collection; diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java new file mode 100644 index 0000000000000..2c7576a3db1db --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/DeleteDataStreamTransportActionTests.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.Index; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInProgressException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.cluster.DataStreamTestHelper.getClusterStateWithDataStreams; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DeleteDataStreamTransportActionTests extends ESTestCase { + + public void testDeleteDataStream() { + final String dataStreamName = "my-data-stream"; + final List otherIndices = randomSubsetOf(Arrays.asList("foo", "bar", "baz")); + + ClusterState cs = getClusterStateWithDataStreams(Collections.singletonList(new Tuple<>(dataStreamName, 2)), otherIndices); + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(0)); + assertThat(newState.metadata().indices().size(), equalTo(otherIndices.size())); + for (String indexName : otherIndices) { + assertThat(newState.metadata().indices().get(indexName).getIndex().getName(), equalTo(indexName)); + } + } + + public void testDeleteMultipleDataStreams() { + String[] dataStreamNames = { "foo", "bar", "baz", "eggplant" }; + ClusterState cs = getClusterStateWithDataStreams( + Arrays.asList( + new Tuple<>(dataStreamNames[0], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[1], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[2], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[3], randomIntBetween(1, 3)) + ), + Collections.emptyList() + ); + + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { "ba*", "eggplant" }); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + DataStream remainingDataStream = newState.metadata().dataStreams().get(dataStreamNames[0]); + assertNotNull(remainingDataStream); + assertThat(newState.metadata().indices().size(), equalTo(remainingDataStream.getIndices().size())); + for (Index i : remainingDataStream.getIndices()) { + assertThat(newState.metadata().indices().get(i.getName()).getIndex(), equalTo(i)); + } + } + + public void testDeleteSnapshottingDataStream() { + final String dataStreamName = "my-data-stream1"; + final String dataStreamName2 = "my-data-stream2"; + final List otherIndices = randomSubsetOf(Arrays.asList("foo", "bar", "baz")); + + ClusterState cs = getClusterStateWithDataStreams( + Arrays.asList(new Tuple<>(dataStreamName, 2), new Tuple<>(dataStreamName2, 2)), + otherIndices + ); + SnapshotsInProgress snapshotsInProgress = SnapshotsInProgress.of( + Arrays.asList(createEntry(dataStreamName, "repo1", false), createEntry(dataStreamName2, "repo2", true)) + ); + ClusterState snapshotCs = ClusterState.builder(cs).putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress).build(); + + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); + SnapshotInProgressException e = expectThrows( + SnapshotInProgressException.class, + () -> DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), snapshotCs, req) + ); + + assertThat( + e.getMessage(), + equalTo( + "Cannot delete data streams that are being snapshotted: [my-data-stream1]. Try again after " + + "snapshot finishes or cancel the currently running snapshot." + ) + ); + } + + private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo, boolean partial) { + return new SnapshotsInProgress.Entry( + new Snapshot(repo, new SnapshotId("", "")), + false, + partial, + SnapshotsInProgress.State.STARTED, + Collections.emptyList(), + Collections.singletonList(dataStreamName), + 0, + 1, + ImmutableOpenMap.of(), + null, + null, + null + ); + } + + public void testDeleteNonexistentDataStream() { + final String dataStreamName = "my-data-stream"; + String[] dataStreamNames = { "foo", "bar", "baz", "eggplant" }; + ClusterState cs = getClusterStateWithDataStreams( + Arrays.asList( + new Tuple<>(dataStreamNames[0], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[1], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[2], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[3], randomIntBetween(1, 3)) + ), + Collections.emptyList() + ); + DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); + ClusterState newState = DeleteDataStreamTransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(cs.metadata().dataStreams().size())); + assertThat( + newState.metadata().dataStreams().keySet(), + containsInAnyOrder(cs.metadata().dataStreams().keySet().toArray(Strings.EMPTY_ARRAY)) + ); + } + + @SuppressWarnings("unchecked") + private static MetadataDeleteIndexService getMetadataDeleteIndexService() { + MetadataDeleteIndexService s = mock(MetadataDeleteIndexService.class); + when(s.deleteIndices(any(ClusterState.class), any(Set.class))).thenAnswer(mockInvocation -> { + ClusterState currentState = (ClusterState) mockInvocation.getArguments()[0]; + Set indices = (Set) mockInvocation.getArguments()[1]; + + final Metadata.Builder b = Metadata.builder(currentState.metadata()); + for (Index index : indices) { + b.remove(index.getName()); + } + + return ClusterState.builder(currentState).metadata(b.build()).build(); + }); + + return s; + } + +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportActionTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportActionTests.java new file mode 100644 index 0000000000000..6ead7c103cb31 --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/action/GetDataStreamsTransportActionTests.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.action; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.action.GetDataStreamAction; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.cluster.DataStreamTestHelper.getClusterStateWithDataStreams; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class GetDataStreamsTransportActionTests extends ESTestCase { + + public void testGetDataStream() { + final String dataStreamName = "my-data-stream"; + ClusterState cs = getClusterStateWithDataStreams( + Collections.singletonList(new Tuple<>(dataStreamName, 1)), + Collections.emptyList() + ); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamName }); + List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamName)); + } + + public void testGetDataStreamsWithWildcards() { + final String[] dataStreamNames = { "my-data-stream", "another-data-stream" }; + ClusterState cs = getClusterStateWithDataStreams( + Arrays.asList(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), + Collections.emptyList() + ); + + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamNames[1].substring(0, 5) + "*" }); + List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + + req = new GetDataStreamAction.Request(new String[] { "*" }); + dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(2)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); + + req = new GetDataStreamAction.Request((String[]) null); + dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(2)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); + + req = new GetDataStreamAction.Request(new String[] { "matches-none*" }); + dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(0)); + } + + public void testGetDataStreamsWithoutWildcards() { + final String[] dataStreamNames = { "my-data-stream", "another-data-stream" }; + ClusterState cs = getClusterStateWithDataStreams( + Arrays.asList(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), + Collections.emptyList() + ); + + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamNames[0], dataStreamNames[1] }); + List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(2)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); + + req = new GetDataStreamAction.Request(new String[] { dataStreamNames[1] }); + dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + + req = new GetDataStreamAction.Request(new String[] { dataStreamNames[0] }); + dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[0])); + + GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(new String[] { "foo" }); + IndexNotFoundException e = expectThrows( + IndexNotFoundException.class, + () -> GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req2) + ); + assertThat(e.getMessage(), containsString("no such index [foo]")); + } + + public void testGetNonexistentDataStream() { + final String dataStreamName = "my-data-stream"; + ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[] { dataStreamName }); + IndexNotFoundException e = expectThrows( + IndexNotFoundException.class, + () -> GetDataStreamsTransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req) + ); + assertThat(e.getMessage(), containsString("no such index [" + dataStreamName + "]")); + } + +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java index d0cde92f87343..494ce74100862 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -3,26 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.elasticsearch.xpack.datastreams.mapper; import org.elasticsearch.common.Strings; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle index 2ebb8eb9a9f5e..81a9f198f4b97 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/build.gradle @@ -8,6 +8,7 @@ dependencies { testImplementation project(path: xpackModule('ml')) testImplementation project(path: xpackModule('ml'), configuration: 'testArtifacts') testImplementation project(path: ':modules:ingest-common') + testImplementation project(path: xpackModule('data-streams')) } // location for keys and certificates diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 975c983cd3bde..3e131d2b17bdf 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.ml.integration; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.action.DeleteDataStreamAction; import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; import org.elasticsearch.xpack.core.ilm.LifecycleAction; @@ -68,6 +69,7 @@ import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.TokenMetadata; import org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import org.elasticsearch.xpack.ilm.IndexLifecycle; import java.io.IOException; @@ -85,6 +87,7 @@ import static org.elasticsearch.test.XContentTestUtils.convertToMap; import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.is; /** @@ -109,7 +112,8 @@ protected Collection> nodePlugins() { // This is to reduce log spam MockPainlessScriptEngine.TestPlugin.class, // ILM is required for .ml-state template index settings - IndexLifecycle.class); + IndexLifecycle.class, + DataStreamsPlugin.class); } @Override @@ -119,7 +123,8 @@ protected Collection> transportClientPlugins() { Netty4Plugin.class, ReindexPlugin.class, // ILM is required for .ml-state template index settings - IndexLifecycle.class); + IndexLifecycle.class, + DataStreamsPlugin.class); } @Override @@ -149,6 +154,7 @@ protected Settings externalClusterClientSettings() { protected void cleanUp() { setUpgradeModeTo(false); + deleteAllDataStreams(); cleanUpResources(); waitForPendingTasks(); } @@ -289,6 +295,14 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); } + protected static void deleteAllDataStreams() { + AcknowledgedResponse response = client().execute( + DeleteDataStreamAction.INSTANCE, + new DeleteDataStreamAction.Request(new String[]{"*"}) + ).actionGet(); + assertAcked(response); + } + public static class MockPainlessScriptEngine extends MockScriptEngine { public static final String NAME = "painless"; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 68ed142af0f97..62d3aa6fa3ca8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -15,7 +15,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.elasticsearch.xpack.core.action.CreateDataStreamAction; import org.elasticsearch.action.bulk.BulkItemRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.bulk.TransportShardBulkAction;