From 0cfb0e99471ad291a47cffb37aab50a193c08c90 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Mon, 5 Aug 2024 12:06:52 +0530 Subject: [PATCH 01/12] Implementing pagination for _cat/indices Signed-off-by: Harsh Garg --- .../java/org/opensearch/common/Table.java | 68 +++++++ .../rest/action/cat/RestIndicesAction.java | 131 ++++++++++++- .../opensearch/rest/action/cat/RestTable.java | 20 +- .../IndexBasedPaginationStrategy.java | 183 ++++++++++++++++++ .../rest/pagination/PaginationStrategy.java | 37 ++++ .../rest/pagination/package-info.java | 12 ++ .../action/cat/RestIndicesActionTests.java | 2 +- 7 files changed, 445 insertions(+), 8 deletions(-) create mode 100644 server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/package-info.java diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index da14f628efa0f..a48cfc1daab81 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -43,6 +43,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import reactor.util.annotation.NonNull; + import static java.util.Collections.emptyMap; /** @@ -59,9 +61,18 @@ public class Table { private List currentCells; private boolean inHeaders = false; private boolean withTime = false; + private PaginationMetadata paginationMetadata = new PaginationMetadata(false, null, null, null); public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; + public Table() {} + + public Table(@Nullable PaginationMetadata paginationMetadata) { + if (paginationMetadata != null) { + this.paginationMetadata = paginationMetadata; + } + } + public Table startHeaders() { inHeaders = true; currentCells = new ArrayList<>(); @@ -230,6 +241,22 @@ public Map getAliasMap() { return headerAliasMap; } + public boolean isPaginated() { + return paginationMetadata.isResponsePaginated; + } + + public String getPaginatedElement() { + return paginationMetadata.paginatedElement; + } + + public String getNextToken() { + return paginationMetadata.nextToken; + } + + public String getPreviousToken() { + return paginationMetadata.previousToken; + } + /** * Cell in a table * @@ -254,4 +281,45 @@ public Cell(Object value, Map attr) { this.attr = attr; } } + + /** + * Pagination metadata for a table. + * + * @opensearch.internal + */ + public static class PaginationMetadata { + + /** + * boolean denoting whether the table is paginated or not. + */ + public final boolean isResponsePaginated; + + /** + * String denoting the element which is being paginated (for e.g. shards, indices..). + */ + public final String paginatedElement; + + /** + * String denoting the next_token of paginated response, which will be used to fetch next page (if any). + */ + public final String nextToken; + + /** + * String denoting the previous_token of paginated response, which will be used to fetch previous page (if any). + */ + public final String previousToken; + + public PaginationMetadata( + @NonNull boolean isResponsePaginated, + @Nullable String paginatedElement, + @Nullable String nextToken, + @Nullable String previousToken + ) { + this.isResponsePaginated = isResponsePaginated; + assert !isResponsePaginated || paginatedElement != null : "paginatedElement must be specified for a table which is paginated"; + this.paginatedElement = paginatedElement; + this.nextToken = nextToken; + this.previousToken = previousToken; + } + } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 9dc711f804144..7dc97c8cc6e38 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -61,6 +61,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; import java.time.Instant; import java.time.ZoneOffset; @@ -95,10 +96,18 @@ public class RestIndicesAction extends AbstractCatAction { "Parameter [master_timeout] is deprecated and will be removed in 3.0. To support inclusive language, please use [cluster_manager_timeout] instead."; private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; + private static final String DEFAULT_CAT_INDICES_PAGE_SIZE_STRING = "1000"; @Override public List routes() { - return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); + return unmodifiableList( + asList( + new Route(GET, "/_cat/indices"), + new Route(GET, "/_cat/indices/{index}"), + new Route(GET, "/_cat/V2/indices"), + new Route(GET, "/_cat/V2/indices/{index}") + ) + ); } @Override @@ -131,9 +140,13 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli } clusterManagerTimeout = request.paramAsTime("master_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); } + // Check for a paginated query. + if (request.path().contains("/V2/indices")) { + return doPaginatedCatRequest(request, client, clusterManagerTimeout, indices); + } + final TimeValue clusterManagerNodeTimeout = clusterManagerTimeout; final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); - return channel -> { final ActionListener listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { @Override @@ -205,6 +218,91 @@ public void onFailure(final Exception e) { }; } + public RestChannelConsumer doPaginatedCatRequest( + final RestRequest request, + final NodeClient client, + final TimeValue clusterManagerNodeTimeout, + final String[] indices + ) { + final boolean local = request.paramAsBoolean("local", false); + final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); + final String requestedToken = request.param("next_token"); + IndexBasedPaginationStrategy.validateRequestedRequest(requestedToken); + final int pageSize = Integer.parseInt(request.param("max_page_size", DEFAULT_CAT_INDICES_PAGE_SIZE_STRING)); + final boolean latestIndicesFirst = request.paramAsBoolean("latest_indices_first", false); + + return channel -> { + final ActionListener
listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { + @Override + public RestResponse buildResponse(final Table table) throws Exception { + return RestTable.buildResponse(table, channel); + } + }); + + // Fetch all the indices from clusterStateRequest for a paginated query. + sendClusterStateRequest( + indices, + IndicesOptions.lenientExpandHidden(), + local, + clusterManagerNodeTimeout, + client, + new ActionListener() { + @Override + public void onResponse(final ClusterStateResponse clusterStateResponse) { + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy( + requestedToken, + pageSize, + latestIndicesFirst, + clusterStateResponse.getState() + ); + + final GroupedActionListener groupedListener = createGroupedListener( + request, + 4, + listener, + new Table.PaginationMetadata( + true, + "indices", + paginationStrategy.getNextToken(), + paginationStrategy.getPreviousToken() + ) + ); + groupedListener.onResponse(clusterStateResponse); + final String[] indicesToBeQueried = paginationStrategy.getPageElements().toArray(new String[0]); + sendGetSettingsRequest( + indicesToBeQueried, + IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()), + local, + clusterManagerNodeTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + sendIndicesStatsRequest( + indicesToBeQueried, + IndicesOptions.lenientExpandHidden(), + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + sendClusterHealthRequest( + indicesToBeQueried, + IndicesOptions.lenientExpandHidden(), + local, + clusterManagerNodeTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } + + @Override + public void onFailure(final Exception e) { + listener.onFailure(e); + } + } + ); + }; + } + /** * We're using the Get Settings API here to resolve the authorized indices for the user. * This is because the Cluster State and Cluster Health APIs do not filter output based @@ -288,6 +386,15 @@ private GroupedActionListener createGroupedListener( final RestRequest request, final int size, final ActionListener
listener + ) { + return createGroupedListener(request, size, listener, null); + } + + private GroupedActionListener createGroupedListener( + final RestRequest request, + final int size, + final ActionListener
listener, + final Table.PaginationMetadata paginationMetadata ) { return new GroupedActionListener<>(new ActionListener>() { @Override @@ -311,7 +418,14 @@ public void onResponse(final Collection responses) { IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); Map indicesStats = statsResponse.getIndices(); - Table responseTable = buildTable(request, indicesSettings, indicesHealths, indicesStats, indicesStates); + Table responseTable = buildTable( + request, + indicesSettings, + indicesHealths, + indicesStats, + indicesStates, + paginationMetadata + ); listener.onResponse(responseTable); } catch (Exception e) { onFailure(e); @@ -340,7 +454,11 @@ protected Set responseParams() { @Override protected Table getTableWithHeader(final RestRequest request) { - Table table = new Table(); + return getTableWithHeader(request, null); + } + + protected Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { + Table table = new Table(paginationMetadata); table.startHeaders(); table.addCell("health", "alias:h;desc:current health status"); table.addCell("status", "alias:s;desc:open/close status"); @@ -709,11 +827,12 @@ Table buildTable( final Map indicesSettings, final Map indicesHealths, final Map indicesStats, - final Map indicesMetadatas + final Map indicesMetadatas, + final Table.PaginationMetadata paginationMetadata ) { final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request); + final Table table = getTableWithHeader(request, paginationMetadata); indicesSettings.forEach((indexName, settings) -> { if (indicesMetadatas.containsKey(indexName) == false) { diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 4f1090b163ee6..88cc9f3788458 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -88,7 +88,15 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel XContentBuilder builder = channel.newBuilder(); List displayHeaders = buildDisplayHeaders(table, request); - builder.startArray(); + if (table.isPaginated()) { + assert table.getPaginatedElement() != null : "Paginated element is required in-case nextToken is not null"; + builder.startObject(); + builder.field("previous_token", table.getPreviousToken()); + builder.field("next_token", table.getNextToken()); + builder.startArray(table.getPaginatedElement()); + } else { + builder.startArray(); + } List rowOrder = getRowOrder(table, request); for (Integer row : rowOrder) { builder.startObject(); @@ -98,6 +106,9 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel builder.endObject(); } builder.endArray(); + if (table.isPaginated()) { + builder.endObject(); + } return new BytesRestResponse(RestStatus.OK, builder); } @@ -136,6 +147,13 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann } out.append("\n"); } + // Adding a nextToken row, post an empty line, in the response if the table is paginated. + if (table.isPaginated()) { + out.append("previous_token" + " " + table.getPreviousToken()); + out.append("\n"); + out.append("next_token" + " " + table.getNextToken()); + out.append("\n"); + } out.close(); return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOut.bytes()); } diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java new file mode 100644 index 0000000000000..b09e2fd8a65aa --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -0,0 +1,183 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; + +import java.util.Base64; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * @opensearch.internal + */ +public class IndexBasedPaginationStrategy implements PaginationStrategy { + + private final String nextToken; + private final String previousToken; + private final List pageElements; + + private static final String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = + "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; + + public IndexBasedPaginationStrategy(String requestedToken, int maxPageSize, boolean latestIndicesFirst, ClusterState clusterState) { + // validate the requestedToken again, if the rest layer didn't do so. + validateRequestedRequest(requestedToken); + requestedToken = requestedToken == null ? null : new String(Base64.getDecoder().decode(requestedToken), UTF_8); + // Get sorted list of indices from metadata and filter out the required number of indices + List sortedIndicesList = getListOfIndicesSortedByCreateTime(clusterState, latestIndicesFirst, requestedToken); + final int newPageStartIndexNumber = getNewPageIndexStartNumber(requestedToken, sortedIndicesList, clusterState); // inclusive + int newPageEndIndexNumber = Math.min(newPageStartIndexNumber + maxPageSize, sortedIndicesList.size()); // exclusive + this.pageElements = sortedIndicesList.subList(newPageStartIndexNumber, newPageEndIndexNumber); + + // Generate the next_token which is to be passed in the response. + // NextToken = "IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + QueryStartTime + $ + // NameOfLastRespondedIndex" -> (1$12345678$12345678$testIndex) + long queryStartTime = requestedToken == null + ? clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() + : Long.parseLong(requestedToken.split("\\$")[2]); + + int previousPageStartIndexNumber = Math.max(newPageStartIndexNumber - maxPageSize, 0); + this.previousToken = newPageStartIndexNumber <= 0 + ? null + : Base64.getEncoder() + .encodeToString( + (previousPageStartIndexNumber + + "$" + + clusterState.metadata() + .indices() + .get(sortedIndicesList.get(Math.max(previousPageStartIndexNumber - 1, 0))) + .getCreationDate() + + "$" + + queryStartTime + + "$" + + sortedIndicesList.get(Math.max(previousPageStartIndexNumber - 1, 0))).getBytes(UTF_8) + ); + + this.nextToken = newPageEndIndexNumber >= sortedIndicesList.size() + ? null + : Base64.getEncoder() + .encodeToString( + (newPageEndIndexNumber + + "$" + + clusterState.metadata().indices().get(sortedIndicesList.get(newPageEndIndexNumber - 1)).getCreationDate() + + "$" + + queryStartTime + + "$" + + sortedIndicesList.get(newPageEndIndexNumber - 1)).getBytes(UTF_8) + ); + } + + @Override + public String getNextToken() { + return nextToken; + } + + @Override + public String getPreviousToken() { + return previousToken; + } + + @Override + public List getPageElements() { + return pageElements; + } + + private List getListOfIndicesSortedByCreateTime( + final ClusterState clusterState, + boolean latestIndicesFirst, + String requestedToken + ) { + long latestValidIndexCreateTime = requestedToken == null ? Long.MAX_VALUE : Long.parseLong(requestedToken.split("\\$")[2]); + // Filter out the indices which have been created after the latest index which was present when paginated query started + List indicesList = clusterState.getRoutingTable() + .getIndicesRouting() + .keySet() + .stream() + .filter(index -> (latestValidIndexCreateTime - clusterState.metadata().indices().get(index).getCreationDate()) >= 0) + .collect(Collectors.toList()); + // Sort the indices list based on their creation timestamps + indicesList.sort((index1, index2) -> { + Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); + Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); + if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { + return latestIndicesFirst == true ? index2.compareTo(index1) : index1.compareTo(index2); + } + return latestIndicesFirst == true + ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) + : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); + }); + return indicesList; + } + + private int getNewPageIndexStartNumber( + final String nextTokenInRequest, + final List sortedIndicesList, + final ClusterState clusterState + ) { + if (Objects.isNull(nextTokenInRequest)) { + return 0; + } + final String[] nextTokenElements = nextTokenInRequest.split("\\$"); + int newPageStartIndexNumber = Math.min(Integer.parseInt(nextTokenElements[0]), sortedIndicesList.size()); + long lastIndexCreationTime = Long.parseLong(nextTokenElements[1]); + String lastIndexName = nextTokenElements[3]; + if (newPageStartIndexNumber > 0 && !Objects.equals(sortedIndicesList.get(newPageStartIndexNumber - 1), lastIndexName)) { + // case denoting an already responded index has been deleted while the paginated queries are being executed + // find the index whose creation time is just after the index which was last responded + newPageStartIndexNumber--; + while (newPageStartIndexNumber > 0) { + if (clusterState.metadata() + .indices() + .get(sortedIndicesList.get(newPageStartIndexNumber - 1)) + .getCreationDate() < lastIndexCreationTime) { + break; + } + newPageStartIndexNumber--; + } + } + return newPageStartIndexNumber; + } + + /** + * Performs simple validations on token received in the request which was generated using {@link IndexBasedPaginationStrategy}. The token should be base64 encoded, and should contain the expected number of elements separated by "$". The timestamps should also be a valid long. + * @param requestedToken + */ + public static void validateRequestedRequest(String requestedToken) { + if (Objects.isNull(requestedToken)) { + return; + } + try { + requestedToken = new String(Base64.getDecoder().decode(requestedToken), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + + final String[] requestedTokenElements = requestedToken.split("\\$"); + if (requestedTokenElements.length != 4) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + + try { + int newPageStartIndexNumber = Integer.parseInt(requestedTokenElements[0]); + long lastSentIndexCreationTime = Long.parseLong(requestedTokenElements[1]); + long queryStartTime = Long.parseLong(requestedTokenElements[2]); + if (newPageStartIndexNumber < 0 || lastSentIndexCreationTime < 0 || queryStartTime < 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java new file mode 100644 index 0000000000000..e86d9db5dbb3e --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import java.util.List; + +/** + * Interface to be implemented by any strategy getting used for paginating rest responses. + * + * @opensearch.internal + */ +public interface PaginationStrategy { + + /** + * + * @return Base64 encoded string, which can be used to fetch next page of response. + */ + String getNextToken(); + + /** + * + * @return Base64 encoded string, which can be used to fetch previous page. + */ + String getPreviousToken(); + + /** + * + * @return List of elements to be displayed for the current page. + */ + List getPageElements(); +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/package-info.java b/server/src/main/java/org/opensearch/rest/pagination/package-info.java new file mode 100644 index 0000000000000..324b8a6c46f88 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Exposes utilities for Rest actions to paginate responses. + */ +package org.opensearch.rest.pagination; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 96b1c75371697..48f0b5e04fffe 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -138,7 +138,7 @@ public void testBuildTable() { } final RestIndicesAction action = new RestIndicesAction(); - final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas); + final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas, null); // now, verify the table is correct List headers = table.getHeaders(); From d6657226e0619edaf4c8e8f5b699cdce949c3998 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Thu, 29 Aug 2024 16:07:25 +0530 Subject: [PATCH 02/12] Adding RestIndicesListAction along with RestListAction changes Signed-off-by: Harsh Garg --- .../org/opensearch/action/ActionModule.java | 10 + .../java/org/opensearch/common/Table.java | 19 +- .../rest/action/cat/RestIndicesAction.java | 1545 ++++++++--------- .../opensearch/rest/action/cat/RestTable.java | 5 +- .../rest/action/list/AbstractListAction.java | 77 + .../action/list/RestIndicesListAction.java | 175 ++ .../rest/action/list/RestListAction.java | 58 + .../rest/action/list/package-info.java | 12 + .../IndexBasedPaginationStrategy.java | 208 +-- .../opensearch/rest/pagination/PageToken.java | 19 + .../rest/pagination/PaginationStrategy.java | 12 +- .../action/cat/RestIndicesActionTests.java | 10 +- 12 files changed, 1210 insertions(+), 940 deletions(-) create mode 100644 server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/RestListAction.java create mode 100644 server/src/main/java/org/opensearch/rest/action/list/package-info.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageToken.java diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 16c15f553951c..e1dc67d7af7e0 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -456,6 +456,9 @@ import org.opensearch.rest.action.ingest.RestGetPipelineAction; import org.opensearch.rest.action.ingest.RestPutPipelineAction; import org.opensearch.rest.action.ingest.RestSimulatePipelineAction; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.action.list.RestListAction; import org.opensearch.rest.action.search.RestClearScrollAction; import org.opensearch.rest.action.search.RestCountAction; import org.opensearch.rest.action.search.RestCreatePitAction; @@ -793,9 +796,12 @@ private ActionFilters setupActionFilters(List actionPlugins) { public void initRestHandlers(Supplier nodesInCluster) { List catActions = new ArrayList<>(); + List listActions = new ArrayList<>(); Consumer registerHandler = handler -> { if (handler instanceof AbstractCatAction) { catActions.add((AbstractCatAction) handler); + } else if (handler instanceof AbstractListAction) { + listActions.add((AbstractListAction) handler); } restController.registerHandler(handler); }; @@ -968,6 +974,9 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestSnapshotAction()); registerHandler.accept(new RestTemplatesAction()); + // LIST API + registerHandler.accept(new RestIndicesListAction()); + // Point in time API registerHandler.accept(new RestCreatePitAction()); registerHandler.accept(new RestDeletePitAction()); @@ -999,6 +1008,7 @@ public void initRestHandlers(Supplier nodesInCluster) { } } registerHandler.accept(new RestCatAction(catActions)); + registerHandler.accept(new RestListAction(listActions)); registerHandler.accept(new RestDecommissionAction()); registerHandler.accept(new RestGetDecommissionStateAction()); registerHandler.accept(new RestRemoteStoreStatsAction()); diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index a48cfc1daab81..a2d0e7990dac2 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -61,7 +61,7 @@ public class Table { private List currentCells; private boolean inHeaders = false; private boolean withTime = false; - private PaginationMetadata paginationMetadata = new PaginationMetadata(false, null, null, null); + private PaginationMetadata paginationMetadata = new PaginationMetadata(false, null, null); public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; @@ -253,10 +253,6 @@ public String getNextToken() { return paginationMetadata.nextToken; } - public String getPreviousToken() { - return paginationMetadata.previousToken; - } - /** * Cell in a table * @@ -304,22 +300,11 @@ public static class PaginationMetadata { */ public final String nextToken; - /** - * String denoting the previous_token of paginated response, which will be used to fetch previous page (if any). - */ - public final String previousToken; - - public PaginationMetadata( - @NonNull boolean isResponsePaginated, - @Nullable String paginatedElement, - @Nullable String nextToken, - @Nullable String previousToken - ) { + public PaginationMetadata(@NonNull boolean isResponsePaginated, @Nullable String paginatedElement, @Nullable String nextToken) { this.isResponsePaginated = isResponsePaginated; assert !isResponsePaginated || paginatedElement != null : "paginatedElement must be specified for a table which is paginated"; this.paginatedElement = paginatedElement; this.nextToken = nextToken; - this.previousToken = previousToken; } } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 7dc97c8cc6e38..945834107f904 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -61,7 +61,6 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; -import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; import java.time.Instant; import java.time.ZoneOffset; @@ -90,24 +89,15 @@ */ public class RestIndicesAction extends AbstractCatAction { - private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class); private static final String MASTER_TIMEOUT_DEPRECATED_MESSAGE = "Parameter [master_timeout] is deprecated and will be removed in 3.0. To support inclusive language, please use [cluster_manager_timeout] instead."; private static final String DUPLICATE_PARAMETER_ERROR_MESSAGE = "Please only use one of the request parameters [master_timeout, cluster_manager_timeout]."; - private static final String DEFAULT_CAT_INDICES_PAGE_SIZE_STRING = "1000"; @Override public List routes() { - return unmodifiableList( - asList( - new Route(GET, "/_cat/indices"), - new Route(GET, "/_cat/indices/{index}"), - new Route(GET, "/_cat/V2/indices"), - new Route(GET, "/_cat/V2/indices/{index}") - ) - ); + return unmodifiableList(asList(new Route(GET, "/_cat/indices"), new Route(GET, "/_cat/indices/{index}"))); } @Override @@ -140,13 +130,9 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli } clusterManagerTimeout = request.paramAsTime("master_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); } - // Check for a paginated query. - if (request.path().contains("/V2/indices")) { - return doPaginatedCatRequest(request, client, clusterManagerTimeout, indices); - } - final TimeValue clusterManagerNodeTimeout = clusterManagerTimeout; final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); + return channel -> { final ActionListener
listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { @Override @@ -155,7 +141,7 @@ public RestResponse buildResponse(final Table table) throws Exception { } }); - sendGetSettingsRequest( + RestIndicesActionCommonUtils.sendGetSettingsRequest( indices, indicesOptions, local, @@ -164,7 +150,11 @@ public RestResponse buildResponse(final Table table) throws Exception { new ActionListener() { @Override public void onResponse(final GetSettingsResponse getSettingsResponse) { - final GroupedActionListener groupedListener = createGroupedListener(request, 4, listener); + final GroupedActionListener groupedListener = RestIndicesActionCommonUtils.createGroupedListener( + request, + 4, + listener + ); groupedListener.onResponse(getSettingsResponse); // The list of indices that will be returned is determined by the indices returned from the Get Settings call. @@ -184,14 +174,14 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve // the // index names with the same indices options that we used for the initial cluster state request (strictExpand). - sendIndicesStatsRequest( + RestIndicesActionCommonUtils.sendIndicesStatsRequest( indices, subRequestIndicesOptions, includeUnloadedSegments, client, ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) ); - sendClusterStateRequest( + RestIndicesActionCommonUtils.sendClusterStateRequest( indices, subRequestIndicesOptions, local, @@ -199,7 +189,7 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { client, ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) ); - sendClusterHealthRequest( + RestIndicesActionCommonUtils.sendClusterHealthRequest( indices, subRequestIndicesOptions, local, @@ -218,896 +208,833 @@ public void onFailure(final Exception e) { }; } - public RestChannelConsumer doPaginatedCatRequest( - final RestRequest request, - final NodeClient client, - final TimeValue clusterManagerNodeTimeout, - final String[] indices - ) { - final boolean local = request.paramAsBoolean("local", false); - final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); - final String requestedToken = request.param("next_token"); - IndexBasedPaginationStrategy.validateRequestedRequest(requestedToken); - final int pageSize = Integer.parseInt(request.param("max_page_size", DEFAULT_CAT_INDICES_PAGE_SIZE_STRING)); - final boolean latestIndicesFirst = request.paramAsBoolean("latest_indices_first", false); + @Override + protected Table getTableWithHeader(final RestRequest request) { + return RestIndicesActionCommonUtils.getTableWithHeader(request, null); + } - return channel -> { - final ActionListener
listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { - @Override - public RestResponse buildResponse(final Table table) throws Exception { - return RestTable.buildResponse(table, channel); - } - }); + private static final Set RESPONSE_PARAMS; - // Fetch all the indices from clusterStateRequest for a paginated query. - sendClusterStateRequest( - indices, - IndicesOptions.lenientExpandHidden(), - local, - clusterManagerNodeTimeout, - client, - new ActionListener() { - @Override - public void onResponse(final ClusterStateResponse clusterStateResponse) { - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy( - requestedToken, - pageSize, - latestIndicesFirst, - clusterStateResponse.getState() - ); + static { + final Set responseParams = new HashSet<>(asList("local", "health")); + responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } - final GroupedActionListener groupedListener = createGroupedListener( + public static class RestIndicesActionCommonUtils { + + private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); + + /** + * We're using the Get Settings API here to resolve the authorized indices for the user. + * This is because the Cluster State and Cluster Health APIs do not filter output based + * on index privileges, so they can't be used to determine which indices are authorized + * or not. On top of this, the Indices Stats API cannot be used either to resolve indices + * as it does not provide information for all existing indices (for example recovering + * indices or non replicated closed indices are not reported in indices stats response). + */ + public static void sendGetSettingsRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + final GetSettingsRequest request = new GetSettingsRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey()); + + client.admin().indices().getSettings(request, listener); + } + + public static void sendClusterStateRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + + final ClusterStateRequest request = new ClusterStateRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + + client.admin().cluster().state(request, listener); + } + + public static void sendClusterHealthRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + + final ClusterHealthRequest request = new ClusterHealthRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + + client.admin().cluster().health(request, listener); + } + + public static void sendIndicesStatsRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean includeUnloadedSegments, + final NodeClient client, + final ActionListener listener + ) { + + final IndicesStatsRequest request = new IndicesStatsRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.all(); + request.includeUnloadedSegments(includeUnloadedSegments); + + client.admin().indices().stats(request, listener); + } + + public static GroupedActionListener createGroupedListener( + final RestRequest request, + final int size, + final ActionListener
listener + ) { + return createGroupedListener(request, size, listener, null); + } + + public static GroupedActionListener createGroupedListener( + final RestRequest request, + final int size, + final ActionListener
listener, + final Table.PaginationMetadata paginationMetadata + ) { + return new GroupedActionListener<>(new ActionListener>() { + @Override + public void onResponse(final Collection responses) { + try { + GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class); + Map indicesSettings = StreamSupport.stream( + Spliterators.spliterator(settingsResponse.getIndexToSettings().entrySet(), 0), + false + ).collect(Collectors.toMap(cursor -> cursor.getKey(), cursor -> cursor.getValue())); + + ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class); + Map indicesStates = StreamSupport.stream( + stateResponse.getState().getMetadata().spliterator(), + false + ).collect(Collectors.toMap(indexMetadata -> indexMetadata.getIndex().getName(), Function.identity())); + + ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class); + Map indicesHealths = healthResponse.getIndices(); + + IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); + Map indicesStats = statsResponse.getIndices(); + + Table responseTable = RestIndicesAction.RestIndicesActionCommonUtils.buildTable( request, - 4, - listener, - new Table.PaginationMetadata( - true, - "indices", - paginationStrategy.getNextToken(), - paginationStrategy.getPreviousToken() - ) - ); - groupedListener.onResponse(clusterStateResponse); - final String[] indicesToBeQueried = paginationStrategy.getPageElements().toArray(new String[0]); - sendGetSettingsRequest( - indicesToBeQueried, - IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()), - local, - clusterManagerNodeTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendIndicesStatsRequest( - indicesToBeQueried, - IndicesOptions.lenientExpandHidden(), - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendClusterHealthRequest( - indicesToBeQueried, - IndicesOptions.lenientExpandHidden(), - local, - clusterManagerNodeTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + indicesSettings, + indicesHealths, + indicesStats, + indicesStates, + paginationMetadata ); + listener.onResponse(responseTable); + } catch (Exception e) { + onFailure(e); } + } - @Override - public void onFailure(final Exception e) { - listener.onFailure(e); - } + @Override + public void onFailure(final Exception e) { + listener.onFailure(e); } + }, size); + } + + public static Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { + Table table = new Table(paginationMetadata); + table.startHeaders(); + table.addCell("health", "alias:h;desc:current health status"); + table.addCell("status", "alias:s;desc:open/close status"); + table.addCell("index", "alias:i,idx;desc:index name"); + table.addCell("uuid", "alias:id,uuid;desc:index uuid"); + table.addCell("pri", "alias:p,shards.primary,shardsPrimary;text-align:right;desc:number of primary shards"); + table.addCell("rep", "alias:r,shards.replica,shardsReplica;text-align:right;desc:number of replica shards"); + table.addCell("docs.count", "alias:dc,docsCount;text-align:right;desc:available docs"); + table.addCell("docs.deleted", "alias:dd,docsDeleted;text-align:right;desc:deleted docs"); + + table.addCell("creation.date", "alias:cd;default:false;desc:index creation date (millisecond value)"); + table.addCell("creation.date.string", "alias:cds;default:false;desc:index creation date (as string)"); + + table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); + table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); + + table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); + table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); + + table.addCell( + "fielddata.memory_size", + "sibling:pri;alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache" ); - }; - } + table.addCell("pri.fielddata.memory_size", "default:false;text-align:right;desc:used fielddata cache"); - /** - * We're using the Get Settings API here to resolve the authorized indices for the user. - * This is because the Cluster State and Cluster Health APIs do not filter output based - * on index privileges, so they can't be used to determine which indices are authorized - * or not. On top of this, the Indices Stats API cannot be used either to resolve indices - * as it does not provide information for all existing indices (for example recovering - * indices or non replicated closed indices are not reported in indices stats response). - */ - private void sendGetSettingsRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - final GetSettingsRequest request = new GetSettingsRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey()); - - client.admin().indices().getSettings(request, listener); - } + table.addCell( + "fielddata.evictions", + "sibling:pri;alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions" + ); + table.addCell("pri.fielddata.evictions", "default:false;text-align:right;desc:fielddata evictions"); - private void sendClusterStateRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - - final ClusterStateRequest request = new ClusterStateRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - - client.admin().cluster().state(request, listener); - } + table.addCell( + "query_cache.memory_size", + "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache" + ); + table.addCell("pri.query_cache.memory_size", "default:false;text-align:right;desc:used query cache"); - private void sendClusterHealthRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - - final ClusterHealthRequest request = new ClusterHealthRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - - client.admin().cluster().health(request, listener); - } + table.addCell( + "query_cache.evictions", + "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions" + ); + table.addCell("pri.query_cache.evictions", "default:false;text-align:right;desc:query cache evictions"); - private void sendIndicesStatsRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean includeUnloadedSegments, - final NodeClient client, - final ActionListener listener - ) { - - final IndicesStatsRequest request = new IndicesStatsRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.all(); - request.includeUnloadedSegments(includeUnloadedSegments); - - client.admin().indices().stats(request, listener); - } + table.addCell( + "request_cache.memory_size", + "sibling:pri;alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache" + ); + table.addCell("pri.request_cache.memory_size", "default:false;text-align:right;desc:used request cache"); - private GroupedActionListener createGroupedListener( - final RestRequest request, - final int size, - final ActionListener
listener - ) { - return createGroupedListener(request, size, listener, null); - } + table.addCell( + "request_cache.evictions", + "sibling:pri;alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions" + ); + table.addCell("pri.request_cache.evictions", "default:false;text-align:right;desc:request cache evictions"); - private GroupedActionListener createGroupedListener( - final RestRequest request, - final int size, - final ActionListener
listener, - final Table.PaginationMetadata paginationMetadata - ) { - return new GroupedActionListener<>(new ActionListener>() { - @Override - public void onResponse(final Collection responses) { - try { - GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class); - Map indicesSettings = StreamSupport.stream( - Spliterators.spliterator(settingsResponse.getIndexToSettings().entrySet(), 0), - false - ).collect(Collectors.toMap(cursor -> cursor.getKey(), cursor -> cursor.getValue())); - - ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class); - Map indicesStates = StreamSupport.stream( - stateResponse.getState().getMetadata().spliterator(), - false - ).collect(Collectors.toMap(indexMetadata -> indexMetadata.getIndex().getName(), Function.identity())); - - ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class); - Map indicesHealths = healthResponse.getIndices(); - - IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); - Map indicesStats = statsResponse.getIndices(); - - Table responseTable = buildTable( - request, - indicesSettings, - indicesHealths, - indicesStats, - indicesStates, - paginationMetadata - ); - listener.onResponse(responseTable); - } catch (Exception e) { - onFailure(e); - } - } + table.addCell( + "request_cache.hit_count", + "sibling:pri;alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit count" + ); + table.addCell("pri.request_cache.hit_count", "default:false;text-align:right;desc:request cache hit count"); - @Override - public void onFailure(final Exception e) { - listener.onFailure(e); - } - }, size); - } + table.addCell( + "request_cache.miss_count", + "sibling:pri;alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss count" + ); + table.addCell("pri.request_cache.miss_count", "default:false;text-align:right;desc:request cache miss count"); - private static final Set RESPONSE_PARAMS; + table.addCell("flush.total", "sibling:pri;alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes"); + table.addCell("pri.flush.total", "default:false;text-align:right;desc:number of flushes"); - static { - final Set responseParams = new HashSet<>(asList("local", "health")); - responseParams.addAll(AbstractCatAction.RESPONSE_PARAMS); - RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); - } + table.addCell( + "flush.total_time", + "sibling:pri;alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush" + ); + table.addCell("pri.flush.total_time", "default:false;text-align:right;desc:time spent in flush"); - @Override - protected Set responseParams() { - return RESPONSE_PARAMS; - } + table.addCell("get.current", "sibling:pri;alias:gc,getCurrent;default:false;text-align:right;desc:number of current get ops"); + table.addCell("pri.get.current", "default:false;text-align:right;desc:number of current get ops"); - @Override - protected Table getTableWithHeader(final RestRequest request) { - return getTableWithHeader(request, null); - } + table.addCell("get.time", "sibling:pri;alias:gti,getTime;default:false;text-align:right;desc:time spent in get"); + table.addCell("pri.get.time", "default:false;text-align:right;desc:time spent in get"); - protected Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { - Table table = new Table(paginationMetadata); - table.startHeaders(); - table.addCell("health", "alias:h;desc:current health status"); - table.addCell("status", "alias:s;desc:open/close status"); - table.addCell("index", "alias:i,idx;desc:index name"); - table.addCell("uuid", "alias:id,uuid;desc:index uuid"); - table.addCell("pri", "alias:p,shards.primary,shardsPrimary;text-align:right;desc:number of primary shards"); - table.addCell("rep", "alias:r,shards.replica,shardsReplica;text-align:right;desc:number of replica shards"); - table.addCell("docs.count", "alias:dc,docsCount;text-align:right;desc:available docs"); - table.addCell("docs.deleted", "alias:dd,docsDeleted;text-align:right;desc:deleted docs"); - - table.addCell("creation.date", "alias:cd;default:false;desc:index creation date (millisecond value)"); - table.addCell("creation.date.string", "alias:cds;default:false;desc:index creation date (as string)"); - - table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); - table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); - - table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); - table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); - - table.addCell( - "fielddata.memory_size", - "sibling:pri;alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache" - ); - table.addCell("pri.fielddata.memory_size", "default:false;text-align:right;desc:used fielddata cache"); - - table.addCell( - "fielddata.evictions", - "sibling:pri;alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions" - ); - table.addCell("pri.fielddata.evictions", "default:false;text-align:right;desc:fielddata evictions"); - - table.addCell( - "query_cache.memory_size", - "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache" - ); - table.addCell("pri.query_cache.memory_size", "default:false;text-align:right;desc:used query cache"); - - table.addCell( - "query_cache.evictions", - "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions" - ); - table.addCell("pri.query_cache.evictions", "default:false;text-align:right;desc:query cache evictions"); - - table.addCell( - "request_cache.memory_size", - "sibling:pri;alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache" - ); - table.addCell("pri.request_cache.memory_size", "default:false;text-align:right;desc:used request cache"); - - table.addCell( - "request_cache.evictions", - "sibling:pri;alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions" - ); - table.addCell("pri.request_cache.evictions", "default:false;text-align:right;desc:request cache evictions"); - - table.addCell( - "request_cache.hit_count", - "sibling:pri;alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit count" - ); - table.addCell("pri.request_cache.hit_count", "default:false;text-align:right;desc:request cache hit count"); - - table.addCell( - "request_cache.miss_count", - "sibling:pri;alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss count" - ); - table.addCell("pri.request_cache.miss_count", "default:false;text-align:right;desc:request cache miss count"); - - table.addCell("flush.total", "sibling:pri;alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes"); - table.addCell("pri.flush.total", "default:false;text-align:right;desc:number of flushes"); - - table.addCell("flush.total_time", "sibling:pri;alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush"); - table.addCell("pri.flush.total_time", "default:false;text-align:right;desc:time spent in flush"); - - table.addCell("get.current", "sibling:pri;alias:gc,getCurrent;default:false;text-align:right;desc:number of current get ops"); - table.addCell("pri.get.current", "default:false;text-align:right;desc:number of current get ops"); - - table.addCell("get.time", "sibling:pri;alias:gti,getTime;default:false;text-align:right;desc:time spent in get"); - table.addCell("pri.get.time", "default:false;text-align:right;desc:time spent in get"); - - table.addCell("get.total", "sibling:pri;alias:gto,getTotal;default:false;text-align:right;desc:number of get ops"); - table.addCell("pri.get.total", "default:false;text-align:right;desc:number of get ops"); - - table.addCell( - "get.exists_time", - "sibling:pri;alias:geti,getExistsTime;default:false;text-align:right;desc:time spent in successful gets" - ); - table.addCell("pri.get.exists_time", "default:false;text-align:right;desc:time spent in successful gets"); - - table.addCell( - "get.exists_total", - "sibling:pri;alias:geto,getExistsTotal;default:false;text-align:right;desc:number of successful gets" - ); - table.addCell("pri.get.exists_total", "default:false;text-align:right;desc:number of successful gets"); - - table.addCell( - "get.missing_time", - "sibling:pri;alias:gmti,getMissingTime;default:false;text-align:right;desc:time spent in failed gets" - ); - table.addCell("pri.get.missing_time", "default:false;text-align:right;desc:time spent in failed gets"); - - table.addCell( - "get.missing_total", - "sibling:pri;alias:gmto,getMissingTotal;default:false;text-align:right;desc:number of failed gets" - ); - table.addCell("pri.get.missing_total", "default:false;text-align:right;desc:number of failed gets"); - - table.addCell( - "indexing.delete_current", - "sibling:pri;alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions" - ); - table.addCell("pri.indexing.delete_current", "default:false;text-align:right;desc:number of current deletions"); - - table.addCell( - "indexing.delete_time", - "sibling:pri;alias:idti,indexingDeleteTime;default:false;text-align:right;desc:time spent in deletions" - ); - table.addCell("pri.indexing.delete_time", "default:false;text-align:right;desc:time spent in deletions"); - - table.addCell( - "indexing.delete_total", - "sibling:pri;alias:idto,indexingDeleteTotal;default:false;text-align:right;desc:number of delete ops" - ); - table.addCell("pri.indexing.delete_total", "default:false;text-align:right;desc:number of delete ops"); - - table.addCell( - "indexing.index_current", - "sibling:pri;alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops" - ); - table.addCell("pri.indexing.index_current", "default:false;text-align:right;desc:number of current indexing ops"); - - table.addCell( - "indexing.index_time", - "sibling:pri;alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing" - ); - table.addCell("pri.indexing.index_time", "default:false;text-align:right;desc:time spent in indexing"); - - table.addCell( - "indexing.index_total", - "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops" - ); - table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops"); - - table.addCell( - "indexing.index_failed", - "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" - ); - table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops"); - - table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges"); - table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges"); - - table.addCell( - "merges.current_docs", - "sibling:pri;alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs" - ); - table.addCell("pri.merges.current_docs", "default:false;text-align:right;desc:number of current merging docs"); - - table.addCell( - "merges.current_size", - "sibling:pri;alias:mcs,mergesCurrentSize;default:false;text-align:right;desc:size of current merges" - ); - table.addCell("pri.merges.current_size", "default:false;text-align:right;desc:size of current merges"); - - table.addCell("merges.total", "sibling:pri;alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops"); - table.addCell("pri.merges.total", "default:false;text-align:right;desc:number of completed merge ops"); - - table.addCell("merges.total_docs", "sibling:pri;alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged"); - table.addCell("pri.merges.total_docs", "default:false;text-align:right;desc:docs merged"); - - table.addCell("merges.total_size", "sibling:pri;alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); - table.addCell("pri.merges.total_size", "default:false;text-align:right;desc:size merged"); - - table.addCell( - "merges.total_time", - "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges" - ); - table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges"); - - table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); - table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes"); - - table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); - table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes"); - - table.addCell( - "refresh.external_total", - "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total external refreshes" - ); - table.addCell("pri.refresh.external_total", "default:false;text-align:right;desc:total external refreshes"); - - table.addCell( - "refresh.external_time", - "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in external refreshes" - ); - table.addCell("pri.refresh.external_time", "default:false;text-align:right;desc:time spent in external refreshes"); - - table.addCell( - "refresh.listeners", - "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners" - ); - table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners"); - - table.addCell( - "search.fetch_current", - "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops" - ); - table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops"); - - table.addCell( - "search.fetch_time", - "sibling:pri;alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase" - ); - table.addCell("pri.search.fetch_time", "default:false;text-align:right;desc:time spent in fetch phase"); - - table.addCell("search.fetch_total", "sibling:pri;alias:sfto,searchFetchTotal;default:false;text-align:right;desc:total fetch ops"); - table.addCell("pri.search.fetch_total", "default:false;text-align:right;desc:total fetch ops"); - - table.addCell( - "search.open_contexts", - "sibling:pri;alias:so,searchOpenContexts;default:false;text-align:right;desc:open search contexts" - ); - table.addCell("pri.search.open_contexts", "default:false;text-align:right;desc:open search contexts"); - - table.addCell( - "search.query_current", - "sibling:pri;alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops" - ); - table.addCell("pri.search.query_current", "default:false;text-align:right;desc:current query phase ops"); - - table.addCell( - "search.query_time", - "sibling:pri;alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase" - ); - table.addCell("pri.search.query_time", "default:false;text-align:right;desc:time spent in query phase"); - - table.addCell( - "search.query_total", - "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops" - ); - table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops"); - table.addCell( - "search.concurrent_query_current", - "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" - ); - table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); - - table.addCell( - "search.concurrent_query_time", - "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" - ); - table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); - - table.addCell( - "search.concurrent_query_total", - "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" - ); - table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); - - table.addCell( - "search.concurrent_avg_slice_count", - "sibling:pri;alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" - ); - table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); - - table.addCell( - "search.scroll_current", - "sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts" - ); - table.addCell("pri.search.scroll_current", "default:false;text-align:right;desc:open scroll contexts"); - - table.addCell( - "search.scroll_time", - "sibling:pri;alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open" - ); - table.addCell("pri.search.scroll_time", "default:false;text-align:right;desc:time scroll contexts held open"); - - table.addCell( - "search.scroll_total", - "sibling:pri;alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts" - ); - table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts"); - - table.addCell( - "search.point_in_time_current", - "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts" - ); - table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open point in time contexts"); - - table.addCell( - "search.point_in_time_time", - "sibling:pri;alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open" - ); - table.addCell("pri.search.point_in_time_time", "default:false;text-align:right;desc:time point in time contexts held open"); - - table.addCell( - "search.point_in_time_total", - "sibling:pri;alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts" - ); - table.addCell("pri.search.point_in_time_total", "default:false;text-align:right;desc:completed point in time contexts"); - - table.addCell("segments.count", "sibling:pri;alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); - table.addCell("pri.segments.count", "default:false;text-align:right;desc:number of segments"); - - table.addCell("segments.memory", "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments"); - table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments"); - - table.addCell( - "segments.index_writer_memory", - "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer" - ); - table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer"); - - table.addCell( - "segments.version_map_memory", - "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map" - ); - table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map"); - - table.addCell( - "segments.fixed_bitset_memory", - "sibling:pri;alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for" - + " nested object field types and type filters for types referred in _parent fields" - ); - table.addCell( - "pri.segments.fixed_bitset_memory", - "default:false;text-align:right;desc:memory used by fixed bit sets for nested object" - + " field types and type filters for types referred in _parent fields" - ); - - table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops"); - table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops"); - - table.addCell("warmer.total", "sibling:pri;alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops"); - table.addCell("pri.warmer.total", "default:false;text-align:right;desc:total warmer ops"); - - table.addCell( - "warmer.total_time", - "sibling:pri;alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers" - ); - table.addCell("pri.warmer.total_time", "default:false;text-align:right;desc:time spent in warmers"); - - table.addCell( - "suggest.current", - "sibling:pri;alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops" - ); - table.addCell("pri.suggest.current", "default:false;text-align:right;desc:number of current suggest ops"); - - table.addCell("suggest.time", "sibling:pri;alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest"); - table.addCell("pri.suggest.time", "default:false;text-align:right;desc:time spend in suggest"); - - table.addCell("suggest.total", "sibling:pri;alias:suto,suggestTotal;default:false;text-align:right;desc:number of suggest ops"); - table.addCell("pri.suggest.total", "default:false;text-align:right;desc:number of suggest ops"); - - table.addCell("memory.total", "sibling:pri;alias:tm,memoryTotal;default:false;text-align:right;desc:total used memory"); - table.addCell("pri.memory.total", "default:false;text-align:right;desc:total user memory"); - - table.addCell("search.throttled", "alias:sth;default:false;desc:indicates if the index is search throttled"); - - table.endHeaders(); - return table; - } + table.addCell("get.total", "sibling:pri;alias:gto,getTotal;default:false;text-align:right;desc:number of get ops"); + table.addCell("pri.get.total", "default:false;text-align:right;desc:number of get ops"); - // package private for testing - Table buildTable( - final RestRequest request, - final Map indicesSettings, - final Map indicesHealths, - final Map indicesStats, - final Map indicesMetadatas, - final Table.PaginationMetadata paginationMetadata - ) { - - final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request, paginationMetadata); - - indicesSettings.forEach((indexName, settings) -> { - if (indicesMetadatas.containsKey(indexName) == false) { - // the index exists in the Get Indices response but is not present in the cluster state: - // it is likely that the index was deleted in the meanwhile, so we ignore it. - return; - } + table.addCell( + "get.exists_time", + "sibling:pri;alias:geti,getExistsTime;default:false;text-align:right;desc:time spent in successful gets" + ); + table.addCell("pri.get.exists_time", "default:false;text-align:right;desc:time spent in successful gets"); - final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); - final IndexMetadata.State indexState = indexMetadata.getState(); - final IndexStats indexStats = indicesStats.get(indexName); - final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); - - final String health; - final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); - if (indexHealth != null) { - health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); - } else if (indexStats != null) { - health = "red*"; - } else { - health = ""; - } + table.addCell( + "get.exists_total", + "sibling:pri;alias:geto,getExistsTotal;default:false;text-align:right;desc:number of successful gets" + ); + table.addCell("pri.get.exists_total", "default:false;text-align:right;desc:number of successful gets"); - if (healthParam != null) { - final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); - boolean skip; + table.addCell( + "get.missing_time", + "sibling:pri;alias:gmti,getMissingTime;default:false;text-align:right;desc:time spent in failed gets" + ); + table.addCell("pri.get.missing_time", "default:false;text-align:right;desc:time spent in failed gets"); + + table.addCell( + "get.missing_total", + "sibling:pri;alias:gmto,getMissingTotal;default:false;text-align:right;desc:number of failed gets" + ); + table.addCell("pri.get.missing_total", "default:false;text-align:right;desc:number of failed gets"); + + table.addCell( + "indexing.delete_current", + "sibling:pri;alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions" + ); + table.addCell("pri.indexing.delete_current", "default:false;text-align:right;desc:number of current deletions"); + + table.addCell( + "indexing.delete_time", + "sibling:pri;alias:idti,indexingDeleteTime;default:false;text-align:right;desc:time spent in deletions" + ); + table.addCell("pri.indexing.delete_time", "default:false;text-align:right;desc:time spent in deletions"); + + table.addCell( + "indexing.delete_total", + "sibling:pri;alias:idto,indexingDeleteTotal;default:false;text-align:right;desc:number of delete ops" + ); + table.addCell("pri.indexing.delete_total", "default:false;text-align:right;desc:number of delete ops"); + + table.addCell( + "indexing.index_current", + "sibling:pri;alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops" + ); + table.addCell("pri.indexing.index_current", "default:false;text-align:right;desc:number of current indexing ops"); + + table.addCell( + "indexing.index_time", + "sibling:pri;alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing" + ); + table.addCell("pri.indexing.index_time", "default:false;text-align:right;desc:time spent in indexing"); + + table.addCell( + "indexing.index_total", + "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops" + ); + table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops"); + + table.addCell( + "indexing.index_failed", + "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" + ); + table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops"); + + table.addCell( + "merges.current", + "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges" + ); + table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges"); + + table.addCell( + "merges.current_docs", + "sibling:pri;alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs" + ); + table.addCell("pri.merges.current_docs", "default:false;text-align:right;desc:number of current merging docs"); + + table.addCell( + "merges.current_size", + "sibling:pri;alias:mcs,mergesCurrentSize;default:false;text-align:right;desc:size of current merges" + ); + table.addCell("pri.merges.current_size", "default:false;text-align:right;desc:size of current merges"); + + table.addCell( + "merges.total", + "sibling:pri;alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops" + ); + table.addCell("pri.merges.total", "default:false;text-align:right;desc:number of completed merge ops"); + + table.addCell("merges.total_docs", "sibling:pri;alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged"); + table.addCell("pri.merges.total_docs", "default:false;text-align:right;desc:docs merged"); + + table.addCell("merges.total_size", "sibling:pri;alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); + table.addCell("pri.merges.total_size", "default:false;text-align:right;desc:size merged"); + + table.addCell( + "merges.total_time", + "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges" + ); + table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges"); + + table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); + table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes"); + + table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); + table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes"); + + table.addCell( + "refresh.external_total", + "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total external refreshes" + ); + table.addCell("pri.refresh.external_total", "default:false;text-align:right;desc:total external refreshes"); + + table.addCell( + "refresh.external_time", + "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in external refreshes" + ); + table.addCell("pri.refresh.external_time", "default:false;text-align:right;desc:time spent in external refreshes"); + + table.addCell( + "refresh.listeners", + "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners" + ); + table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners"); + + table.addCell( + "search.fetch_current", + "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops" + ); + table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops"); + + table.addCell( + "search.fetch_time", + "sibling:pri;alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase" + ); + table.addCell("pri.search.fetch_time", "default:false;text-align:right;desc:time spent in fetch phase"); + + table.addCell( + "search.fetch_total", + "sibling:pri;alias:sfto,searchFetchTotal;default:false;text-align:right;desc:total fetch ops" + ); + table.addCell("pri.search.fetch_total", "default:false;text-align:right;desc:total fetch ops"); + + table.addCell( + "search.open_contexts", + "sibling:pri;alias:so,searchOpenContexts;default:false;text-align:right;desc:open search contexts" + ); + table.addCell("pri.search.open_contexts", "default:false;text-align:right;desc:open search contexts"); + + table.addCell( + "search.query_current", + "sibling:pri;alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops" + ); + table.addCell("pri.search.query_current", "default:false;text-align:right;desc:current query phase ops"); + + table.addCell( + "search.query_time", + "sibling:pri;alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase" + ); + table.addCell("pri.search.query_time", "default:false;text-align:right;desc:time spent in query phase"); + + table.addCell( + "search.query_total", + "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops" + ); + table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops"); + table.addCell( + "search.concurrent_query_current", + "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" + ); + table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); + + table.addCell( + "search.concurrent_query_time", + "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" + ); + table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); + + table.addCell( + "search.concurrent_query_total", + "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" + ); + table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); + + table.addCell( + "search.concurrent_avg_slice_count", + "sibling:pri;alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" + ); + table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); + + table.addCell( + "search.scroll_current", + "sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts" + ); + table.addCell("pri.search.scroll_current", "default:false;text-align:right;desc:open scroll contexts"); + + table.addCell( + "search.scroll_time", + "sibling:pri;alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open" + ); + table.addCell("pri.search.scroll_time", "default:false;text-align:right;desc:time scroll contexts held open"); + + table.addCell( + "search.scroll_total", + "sibling:pri;alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts" + ); + table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts"); + + table.addCell( + "search.point_in_time_current", + "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts" + ); + table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open point in time contexts"); + + table.addCell( + "search.point_in_time_time", + "sibling:pri;alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open" + ); + table.addCell("pri.search.point_in_time_time", "default:false;text-align:right;desc:time point in time contexts held open"); + + table.addCell( + "search.point_in_time_total", + "sibling:pri;alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts" + ); + table.addCell("pri.search.point_in_time_total", "default:false;text-align:right;desc:completed point in time contexts"); + + table.addCell("segments.count", "sibling:pri;alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); + table.addCell("pri.segments.count", "default:false;text-align:right;desc:number of segments"); + + table.addCell( + "segments.memory", + "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments" + ); + table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments"); + + table.addCell( + "segments.index_writer_memory", + "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer" + ); + table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer"); + + table.addCell( + "segments.version_map_memory", + "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map" + ); + table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map"); + + table.addCell( + "segments.fixed_bitset_memory", + "sibling:pri;alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for" + + " nested object field types and type filters for types referred in _parent fields" + ); + table.addCell( + "pri.segments.fixed_bitset_memory", + "default:false;text-align:right;desc:memory used by fixed bit sets for nested object" + + " field types and type filters for types referred in _parent fields" + ); + + table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops"); + table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops"); + + table.addCell("warmer.total", "sibling:pri;alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops"); + table.addCell("pri.warmer.total", "default:false;text-align:right;desc:total warmer ops"); + + table.addCell( + "warmer.total_time", + "sibling:pri;alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers" + ); + table.addCell("pri.warmer.total_time", "default:false;text-align:right;desc:time spent in warmers"); + + table.addCell( + "suggest.current", + "sibling:pri;alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops" + ); + table.addCell("pri.suggest.current", "default:false;text-align:right;desc:number of current suggest ops"); + + table.addCell("suggest.time", "sibling:pri;alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest"); + table.addCell("pri.suggest.time", "default:false;text-align:right;desc:time spend in suggest"); + + table.addCell("suggest.total", "sibling:pri;alias:suto,suggestTotal;default:false;text-align:right;desc:number of suggest ops"); + table.addCell("pri.suggest.total", "default:false;text-align:right;desc:number of suggest ops"); + + table.addCell("memory.total", "sibling:pri;alias:tm,memoryTotal;default:false;text-align:right;desc:total used memory"); + table.addCell("pri.memory.total", "default:false;text-align:right;desc:total user memory"); + + table.addCell("search.throttled", "alias:sth;default:false;desc:indicates if the index is search throttled"); + + table.endHeaders(); + return table; + } + + // package private for testing + public static Table buildTable( + final RestRequest request, + final Map indicesSettings, + final Map indicesHealths, + final Map indicesStats, + final Map indicesMetadatas, + final Table.PaginationMetadata paginationMetadata + ) { + + final String healthParam = request.param("health"); + final Table table = getTableWithHeader(request, paginationMetadata); + + indicesSettings.forEach((indexName, settings) -> { + if (indicesMetadatas.containsKey(indexName) == false) { + // the index exists in the Get Indices response but is not present in the cluster state: + // it is likely that the index was deleted in the meanwhile, so we ignore it. + return; + } + + final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); + final IndexMetadata.State indexState = indexMetadata.getState(); + final IndexStats indexStats = indicesStats.get(indexName); + final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); + + final String health; + final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); if (indexHealth != null) { - // index health is known but does not match the one requested - skip = indexHealth.getStatus() != healthStatusFilter; + health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); + } else if (indexStats != null) { + health = "red*"; } else { - // index health is unknown, skip if we don't explicitly request RED health - skip = ClusterHealthStatus.RED != healthStatusFilter; + health = ""; } - if (skip) { - return; + + if (healthParam != null) { + final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); + boolean skip; + if (indexHealth != null) { + // index health is known but does not match the one requested + skip = indexHealth.getStatus() != healthStatusFilter; + } else { + // index health is unknown, skip if we don't explicitly request RED health + skip = ClusterHealthStatus.RED != healthStatusFilter; + } + if (skip) { + return; + } } - } - final CommonStats primaryStats; - final CommonStats totalStats; + final CommonStats primaryStats; + final CommonStats totalStats; - if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { - // TODO: expose docs stats for replicated closed indices - primaryStats = new CommonStats(); - totalStats = new CommonStats(); - } else { - primaryStats = indexStats.getPrimaries(); - totalStats = indexStats.getTotal(); - } - table.startRow(); - table.addCell(health); - table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); - table.addCell(indexName); - table.addCell(indexMetadata.getIndexUUID()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); + if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { + // TODO: expose docs stats for replicated closed indices + primaryStats = new CommonStats(); + totalStats = new CommonStats(); + } else { + primaryStats = indexStats.getPrimaries(); + totalStats = indexStats.getTotal(); + } + table.startRow(); + table.addCell(health); + table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); + table.addCell(indexName); + table.addCell(indexMetadata.getIndexUUID()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); - table.addCell(indexMetadata.getCreationDate()); - ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); - table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); + table.addCell(indexMetadata.getCreationDate()); + ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); + table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); - table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); - table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); + table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); - table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); - table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); + table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); - table.addCell(totalStats.getTotalMemory()); - table.addCell(primaryStats.getTotalMemory()); + table.addCell(totalStats.getTotalMemory()); + table.addCell(primaryStats.getTotalMemory()); - table.addCell(searchThrottled); + table.addCell(searchThrottled); - table.endRow(); - }); + table.endRow(); + }); - return table; - } + return table; + } + + @SuppressWarnings("unchecked") + private static A extractResponse(final Collection responses, Class c) { + return (A) responses.stream().filter(c::isInstance).findFirst().get(); + } - @SuppressWarnings("unchecked") - private static A extractResponse(final Collection responses, Class c) { - return (A) responses.stream().filter(c::isInstance).findFirst().get(); } + } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 88cc9f3788458..1eecc70b7e213 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -91,7 +91,6 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel if (table.isPaginated()) { assert table.getPaginatedElement() != null : "Paginated element is required in-case nextToken is not null"; builder.startObject(); - builder.field("previous_token", table.getPreviousToken()); builder.field("next_token", table.getNextToken()); builder.startArray(table.getPaginatedElement()); } else { @@ -147,10 +146,8 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann } out.append("\n"); } - // Adding a nextToken row, post an empty line, in the response if the table is paginated. + // Adding a new row for next_token, in the response if the table is paginated. if (table.isPaginated()) { - out.append("previous_token" + " " + table.getPreviousToken()); - out.append("\n"); out.append("next_token" + " " + table.getNextToken()); out.append("\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java new file mode 100644 index 0000000000000..ff32e3c49434f --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.Table; +import org.opensearch.common.io.Streams; +import org.opensearch.common.io.UTF8StreamWriter; +import org.opensearch.core.common.io.stream.BytesStream; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.opensearch.rest.action.cat.RestTable.buildHelpWidths; +import static org.opensearch.rest.action.cat.RestTable.pad; + +/** + * Base Transport action class for _list APIs + * + * @opensearch.api + */ +public abstract class AbstractListAction extends BaseRestHandler { + protected abstract RestChannelConsumer doListRequest(RestRequest request, NodeClient client); + + protected abstract void documentation(StringBuilder sb); + + protected abstract Table getTableWithHeader(RestRequest request); + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + boolean helpWanted = request.paramAsBoolean("help", false); + if (helpWanted) { + return channel -> { + Table table = getTableWithHeader(request); + int[] width = buildHelpWidths(table, request); + BytesStream bytesOutput = Streams.flushOnCloseStream(channel.bytesOutput()); + UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput); + for (Table.Cell cell : table.getHeaders()) { + // need to do left-align always, so create new cells + pad(new Table.Cell(cell.value), width[0], request, out); + out.append(" | "); + pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out); + out.append(" | "); + pad(new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"), width[2], request, out); + out.append("\n"); + } + out.close(); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes())); + }; + } else { + return doListRequest(request, client); + } + } + + static Set RESPONSE_PARAMS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("format", "h", "v", "ts", "pri", "bytes", "size", "time", "s", "timeout")) + ); + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java new file mode 100644 index 0000000000000..ee1a5fc9f1411 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.Table; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.Strings; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.action.cat.RestIndicesAction; +import org.opensearch.rest.action.cat.RestTable; +import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest.DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * _list API action to output indices in pages. + * + * @opensearch.api + */ +public class RestIndicesListAction extends AbstractListAction { + private static final String DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = "1000"; + private static final String PAGINATED_ELEMENT_KEY = "indices"; + + @Override + public List routes() { + return unmodifiableList(asList(new Route(GET, "/_list/indices"), new Route(GET, "/_list/indices/{index}"))); + } + + @Override + public String getName() { + return "list_indices_action"; + } + + @Override + public boolean allowSystemIndexAccessByDefault() { + return true; + } + + @Override + protected void documentation(StringBuilder sb) { + sb.append("/_list/indices\n"); + sb.append("/_list/indices/{index}\n"); + } + + @Override + public RestChannelConsumer doListRequest(final RestRequest request, final NodeClient client) { + final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); + final boolean local = request.paramAsBoolean("local", false); + TimeValue clusterManagerTimeout = request.paramAsTime("cluster_manager_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); + final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); + final String requestedToken = request.param("next_token"); + final int pageSize = Integer.parseInt(request.param("size", DEFAULT_LIST_INDICES_PAGE_SIZE_STRING)); + final String requestedSortOrder = request.param("sort", "ascending"); + + return channel -> { + final ActionListener
listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { + @Override + public RestResponse buildResponse(final Table table) throws Exception { + return RestTable.buildResponse(table, channel); + } + }); + + // Fetch all the indices from clusterStateRequest for a paginated query. + RestIndicesAction.RestIndicesActionCommonUtils.sendClusterStateRequest( + indices, + IndicesOptions.lenientExpandHidden(), + local, + clusterManagerTimeout, + client, + new ActionListener() { + @Override + public void onResponse(final ClusterStateResponse clusterStateResponse) { + try { + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy( + requestedToken == null ? null : new IndexBasedPaginationStrategy.IndexStrategyPageToken(requestedToken), + pageSize, + requestedSortOrder, + clusterStateResponse.getState() + ); + + final GroupedActionListener groupedListener = RestIndicesAction.RestIndicesActionCommonUtils + .createGroupedListener( + request, + 4, + listener, + new Table.PaginationMetadata( + true, + PAGINATED_ELEMENT_KEY, + paginationStrategy.getNextToken() == null + ? null + : paginationStrategy.getNextToken().generateEncryptedToken() + ) + ); + groupedListener.onResponse(clusterStateResponse); + + final String[] indicesToBeQueried = paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]); + RestIndicesAction.RestIndicesActionCommonUtils.sendGetSettingsRequest( + indicesToBeQueried, + IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()), + local, + clusterManagerTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + RestIndicesAction.RestIndicesActionCommonUtils.sendIndicesStatsRequest( + indicesToBeQueried, + IndicesOptions.lenientExpandHidden(), + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + RestIndicesAction.RestIndicesActionCommonUtils.sendClusterHealthRequest( + indicesToBeQueried, + IndicesOptions.lenientExpandHidden(), + local, + clusterManagerTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } catch (Exception e) { + onFailure(e); + } + } + + @Override + public void onFailure(final Exception e) { + listener.onFailure(e); + } + } + ); + }; + + } + + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>(asList("local", "health")); + responseParams.addAll(AbstractListAction.RESPONSE_PARAMS); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } + + @Override + protected Table getTableWithHeader(final RestRequest request) { + return RestIndicesAction.RestIndicesActionCommonUtils.getTableWithHeader(request, null); + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java new file mode 100644 index 0000000000000..4b8551ea7e14a --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.RestRequest; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * Base _list API endpoint + * + * @opensearch.api + */ +public class RestListAction extends BaseRestHandler { + + private static final String LIST = ":‑|"; + private static final String LIST_NL = LIST + "\n"; + private final String HELP; + + public RestListAction(List listActions) { + StringBuilder sb = new StringBuilder(); + sb.append(LIST_NL); + for (AbstractListAction listAction : listActions) { + listAction.documentation(sb); + } + HELP = sb.toString(); + } + + @Override + public List routes() { + return singletonList(new Route(GET, "/_list")); + } + + @Override + public String getName() { + return "list_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP)); + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/package-info.java b/server/src/main/java/org/opensearch/rest/action/list/package-info.java new file mode 100644 index 0000000000000..8d6563ff9b344 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * {@link org.opensearch.rest.RestHandler}s for actions that list out results in chunks of pages. + */ +package org.opensearch.rest.action.list; diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java index b09e2fd8a65aa..25dd50a5ebb39 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -10,6 +10,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; +import org.opensearch.common.Nullable; import java.util.Base64; import java.util.List; @@ -19,120 +20,92 @@ import static java.nio.charset.StandardCharsets.UTF_8; /** + * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. + * The strategy considers create timestamps of indices as the keys to iterate over pages. + * * @opensearch.internal */ public class IndexBasedPaginationStrategy implements PaginationStrategy { - private final String nextToken; - private final String previousToken; - private final List pageElements; + private static final String DESCENDING_SORT_PARAM_VALUE = "descending"; + private final IndexStrategyPageToken nextToken; + private final List indicesFromRequestedToken; private static final String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; - public IndexBasedPaginationStrategy(String requestedToken, int maxPageSize, boolean latestIndicesFirst, ClusterState clusterState) { - // validate the requestedToken again, if the rest layer didn't do so. - validateRequestedRequest(requestedToken); - requestedToken = requestedToken == null ? null : new String(Base64.getDecoder().decode(requestedToken), UTF_8); + public IndexBasedPaginationStrategy( + @Nullable IndexStrategyPageToken requestedToken, + int maxPageSize, + String sortOrder, + ClusterState clusterState + ) { // Get sorted list of indices from metadata and filter out the required number of indices - List sortedIndicesList = getListOfIndicesSortedByCreateTime(clusterState, latestIndicesFirst, requestedToken); + List sortedIndicesList = getListOfIndicesSortedByCreateTime(clusterState, sortOrder, requestedToken); final int newPageStartIndexNumber = getNewPageIndexStartNumber(requestedToken, sortedIndicesList, clusterState); // inclusive int newPageEndIndexNumber = Math.min(newPageStartIndexNumber + maxPageSize, sortedIndicesList.size()); // exclusive - this.pageElements = sortedIndicesList.subList(newPageStartIndexNumber, newPageEndIndexNumber); - - // Generate the next_token which is to be passed in the response. - // NextToken = "IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + QueryStartTime + $ - // NameOfLastRespondedIndex" -> (1$12345678$12345678$testIndex) + this.indicesFromRequestedToken = sortedIndicesList.subList(newPageStartIndexNumber, newPageEndIndexNumber); long queryStartTime = requestedToken == null ? clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() - : Long.parseLong(requestedToken.split("\\$")[2]); - - int previousPageStartIndexNumber = Math.max(newPageStartIndexNumber - maxPageSize, 0); - this.previousToken = newPageStartIndexNumber <= 0 - ? null - : Base64.getEncoder() - .encodeToString( - (previousPageStartIndexNumber - + "$" - + clusterState.metadata() - .indices() - .get(sortedIndicesList.get(Math.max(previousPageStartIndexNumber - 1, 0))) - .getCreationDate() - + "$" - + queryStartTime - + "$" - + sortedIndicesList.get(Math.max(previousPageStartIndexNumber - 1, 0))).getBytes(UTF_8) - ); - - this.nextToken = newPageEndIndexNumber >= sortedIndicesList.size() - ? null - : Base64.getEncoder() - .encodeToString( - (newPageEndIndexNumber - + "$" - + clusterState.metadata().indices().get(sortedIndicesList.get(newPageEndIndexNumber - 1)).getCreationDate() - + "$" - + queryStartTime - + "$" - + sortedIndicesList.get(newPageEndIndexNumber - 1)).getBytes(UTF_8) - ); + : requestedToken.queryStartTime; + IndexStrategyPageToken nextPageToken = new IndexStrategyPageToken( + newPageEndIndexNumber, + clusterState.metadata().indices().get(sortedIndicesList.get(newPageEndIndexNumber - 1)).getCreationDate(), + queryStartTime, + sortedIndicesList.get(newPageEndIndexNumber - 1) + ); + this.nextToken = newPageEndIndexNumber >= sortedIndicesList.size() ? null : nextPageToken; } @Override - public String getNextToken() { + @Nullable + public PageToken getNextToken() { return nextToken; } @Override - public String getPreviousToken() { - return previousToken; - } - - @Override - public List getPageElements() { - return pageElements; + @Nullable + public List getElementsFromRequestedToken() { + return indicesFromRequestedToken; } private List getListOfIndicesSortedByCreateTime( final ClusterState clusterState, - boolean latestIndicesFirst, - String requestedToken + String sortOrder, + IndexStrategyPageToken requestedPageToken ) { - long latestValidIndexCreateTime = requestedToken == null ? Long.MAX_VALUE : Long.parseLong(requestedToken.split("\\$")[2]); - // Filter out the indices which have been created after the latest index which was present when paginated query started - List indicesList = clusterState.getRoutingTable() + long latestValidIndexCreateTime = requestedPageToken == null ? Long.MAX_VALUE : requestedPageToken.queryStartTime; + // Filter out the indices which have been created after the latest index which was present when paginated query started. + // Also, sort the indices list based on their creation timestamps + return clusterState.getRoutingTable() .getIndicesRouting() .keySet() .stream() .filter(index -> (latestValidIndexCreateTime - clusterState.metadata().indices().get(index).getCreationDate()) >= 0) + .sorted((index1, index2) -> { + Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); + Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); + if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { + return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) ? index2.compareTo(index1) : index1.compareTo(index2); + } + return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) + : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); + }) .collect(Collectors.toList()); - // Sort the indices list based on their creation timestamps - indicesList.sort((index1, index2) -> { - Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); - Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); - if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { - return latestIndicesFirst == true ? index2.compareTo(index1) : index1.compareTo(index2); - } - return latestIndicesFirst == true - ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) - : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); - }); - return indicesList; } private int getNewPageIndexStartNumber( - final String nextTokenInRequest, + final IndexStrategyPageToken requestedPageToken, final List sortedIndicesList, final ClusterState clusterState ) { - if (Objects.isNull(nextTokenInRequest)) { + if (Objects.isNull(requestedPageToken)) { return 0; } - final String[] nextTokenElements = nextTokenInRequest.split("\\$"); - int newPageStartIndexNumber = Math.min(Integer.parseInt(nextTokenElements[0]), sortedIndicesList.size()); - long lastIndexCreationTime = Long.parseLong(nextTokenElements[1]); - String lastIndexName = nextTokenElements[3]; - if (newPageStartIndexNumber > 0 && !Objects.equals(sortedIndicesList.get(newPageStartIndexNumber - 1), lastIndexName)) { + int newPageStartIndexNumber = Math.min(requestedPageToken.posToStartPage, sortedIndicesList.size() - 1); + if (newPageStartIndexNumber > 0 + && !Objects.equals(sortedIndicesList.get(newPageStartIndexNumber - 1), requestedPageToken.nameOfLastRespondedIndex)) { // case denoting an already responded index has been deleted while the paginated queries are being executed // find the index whose creation time is just after the index which was last responded newPageStartIndexNumber--; @@ -140,7 +113,7 @@ private int getNewPageIndexStartNumber( if (clusterState.metadata() .indices() .get(sortedIndicesList.get(newPageStartIndexNumber - 1)) - .getCreationDate() < lastIndexCreationTime) { + .getCreationDate() < requestedPageToken.creationTimeOfLastRespondedIndex) { break; } newPageStartIndexNumber--; @@ -150,33 +123,70 @@ private int getNewPageIndexStartNumber( } /** - * Performs simple validations on token received in the request which was generated using {@link IndexBasedPaginationStrategy}. The token should be base64 encoded, and should contain the expected number of elements separated by "$". The timestamps should also be a valid long. - * @param requestedToken + * Token to be used by {@link IndexBasedPaginationStrategy}. + * Token would like: IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + + * QueryStartTime + $ + NameOfLastRespondedIndex */ - public static void validateRequestedRequest(String requestedToken) { - if (Objects.isNull(requestedToken)) { - return; - } - try { - requestedToken = new String(Base64.getDecoder().decode(requestedToken), UTF_8); - } catch (IllegalArgumentException exception) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } + public static class IndexStrategyPageToken implements PageToken { + + private final int posToStartPage; + private final long creationTimeOfLastRespondedIndex; + private final long queryStartTime; + private final String nameOfLastRespondedIndex; + + /** + * Will perform simple validations on token received in the request and initialize the data members. + * The token should be base64 encoded, and should contain the expected number of elements separated by "$". + * The timestamps should also be a valid long. + * + * @param requestedTokenString string denoting the encoded next token requested by the user + */ + public IndexStrategyPageToken(String requestedTokenString) { + Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); + try { + requestedTokenString = new String(Base64.getDecoder().decode(requestedTokenString), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } - final String[] requestedTokenElements = requestedToken.split("\\$"); - if (requestedTokenElements.length != 4) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } + final String[] requestedTokenElements = requestedTokenString.split("\\$"); + if (requestedTokenElements.length != 4) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } - try { - int newPageStartIndexNumber = Integer.parseInt(requestedTokenElements[0]); - long lastSentIndexCreationTime = Long.parseLong(requestedTokenElements[1]); - long queryStartTime = Long.parseLong(requestedTokenElements[2]); - if (newPageStartIndexNumber < 0 || lastSentIndexCreationTime < 0 || queryStartTime < 0) { + try { + this.posToStartPage = Integer.parseInt(requestedTokenElements[0]); + this.creationTimeOfLastRespondedIndex = Long.parseLong(requestedTokenElements[1]); + this.queryStartTime = Long.parseLong(requestedTokenElements[2]); + this.nameOfLastRespondedIndex = requestedTokenElements[3]; + if (posToStartPage < 0 || creationTimeOfLastRespondedIndex < 0 || queryStartTime < 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } - } catch (NumberFormatException exception) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + + public IndexStrategyPageToken( + int indexNumberToStartPageFrom, + long creationTimeOfLastRespondedIndex, + long queryStartTime, + String nameOfLastRespondedIndex + ) { + Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); + this.posToStartPage = indexNumberToStartPageFrom; + this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; + this.queryStartTime = queryStartTime; + this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; + } + + @Override + public String generateEncryptedToken() { + return Base64.getEncoder() + .encodeToString( + (posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex) + .getBytes(UTF_8) + ); } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java new file mode 100644 index 0000000000000..0699ee5acea65 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java @@ -0,0 +1,19 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +/** + * To be implemented by tokens getting returned/generated by {@link PaginationStrategy}. + * + * @opensearch.internal + */ +public interface PageToken { + + String generateEncryptedToken(); +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index e86d9db5dbb3e..c5168101e6a33 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -21,17 +21,11 @@ public interface PaginationStrategy { * * @return Base64 encoded string, which can be used to fetch next page of response. */ - String getNextToken(); + PageToken getNextToken(); /** * - * @return Base64 encoded string, which can be used to fetch previous page. + * @return List of elements fetched corresponding to the store and token received by the strategy. */ - String getPreviousToken(); - - /** - * - * @return List of elements to be displayed for the current page. - */ - List getPageElements(); + List getElementsFromRequestedToken(); } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 48f0b5e04fffe..3e30d000f9c07 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -137,8 +137,14 @@ public void testBuildTable() { } } - final RestIndicesAction action = new RestIndicesAction(); - final Table table = action.buildTable(new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas, null); + final Table table = RestIndicesAction.RestIndicesActionCommonUtils.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + null + ); // now, verify the table is correct List headers = table.getHeaders(); From 60eefd62c373954b1d09526e4d7a58ff2a261ce4 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Tue, 3 Sep 2024 09:12:35 +0530 Subject: [PATCH 03/12] Combining the doList and doCat methods and adding UTs Signed-off-by: Harsh Garg --- .../example/resthandler/ExampleCatAction.java | 2 +- .../org/opensearch/action/ActionModule.java | 11 +- .../rest/action/cat/AbstractCatAction.java | 54 +- .../rest/action/cat/RestAliasAction.java | 2 +- .../rest/action/cat/RestAllocationAction.java | 2 +- .../action/cat/RestCatRecoveryAction.java | 2 +- .../cat/RestCatSegmentReplicationAction.java | 2 +- .../action/cat/RestClusterManagerAction.java | 2 +- .../rest/action/cat/RestCountAction.java | 2 +- .../rest/action/cat/RestFielddataAction.java | 2 +- .../rest/action/cat/RestHealthAction.java | 2 +- .../rest/action/cat/RestIndicesAction.java | 1481 ++++++++--------- .../rest/action/cat/RestNodeAttrsAction.java | 2 +- .../rest/action/cat/RestNodesAction.java | 2 +- .../cat/RestPendingClusterTasksAction.java | 2 +- .../action/cat/RestPitSegmentsAction.java | 2 +- .../rest/action/cat/RestPluginsAction.java | 2 +- .../action/cat/RestRepositoriesAction.java | 2 +- .../rest/action/cat/RestSegmentsAction.java | 2 +- .../rest/action/cat/RestShardsAction.java | 2 +- .../rest/action/cat/RestSnapshotAction.java | 2 +- .../rest/action/cat/RestTasksAction.java | 2 +- .../rest/action/cat/RestTemplatesAction.java | 2 +- .../rest/action/cat/RestThreadPoolAction.java | 2 +- .../rest/action/list/AbstractListAction.java | 77 - .../action/list/RestIndicesListAction.java | 171 +- .../rest/action/list/RestListAction.java | 5 +- .../IndexBasedPaginationStrategy.java | 144 +- .../opensearch/rest/pagination/PageToken.java | 28 + .../rest/pagination/PaginationStrategy.java | 35 + .../opensearch/rest/BaseRestHandlerTests.java | 2 +- .../action/cat/RestIndicesActionTests.java | 4 +- .../IndexBasedPaginationStrategyTests.java | 146 ++ 33 files changed, 1150 insertions(+), 1050 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java create mode 100644 server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java diff --git a/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java b/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java index 06e0cee69e68f..a1a16846988a6 100644 --- a/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java +++ b/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java @@ -83,7 +83,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append(documentation()); } diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index e1dc67d7af7e0..3721e2caf9cd3 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -456,7 +456,6 @@ import org.opensearch.rest.action.ingest.RestGetPipelineAction; import org.opensearch.rest.action.ingest.RestPutPipelineAction; import org.opensearch.rest.action.ingest.RestSimulatePipelineAction; -import org.opensearch.rest.action.list.AbstractListAction; import org.opensearch.rest.action.list.RestIndicesListAction; import org.opensearch.rest.action.list.RestListAction; import org.opensearch.rest.action.search.RestClearScrollAction; @@ -796,12 +795,14 @@ private ActionFilters setupActionFilters(List actionPlugins) { public void initRestHandlers(Supplier nodesInCluster) { List catActions = new ArrayList<>(); - List listActions = new ArrayList<>(); + List listActions = new ArrayList<>(); Consumer registerHandler = handler -> { if (handler instanceof AbstractCatAction) { - catActions.add((AbstractCatAction) handler); - } else if (handler instanceof AbstractListAction) { - listActions.add((AbstractListAction) handler); + if (((AbstractCatAction) handler).isActionPaginated()) { + listActions.add((AbstractCatAction) handler); + } else { + catActions.add((AbstractCatAction) handler); + } } restController.registerHandler(handler); }; diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index 6f4e060363bfb..6a93d10a7a8e4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -40,11 +40,14 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; +import org.opensearch.rest.pagination.PageToken; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Map; +import java.util.Objects; import java.util.Set; import static org.opensearch.rest.action.cat.RestTable.buildHelpWidths; @@ -57,9 +60,11 @@ */ public abstract class AbstractCatAction extends BaseRestHandler { + protected PaginationQueryMetadata paginationQueryMetadata; + protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client); - protected abstract void documentation(StringBuilder sb); + public abstract void documentation(StringBuilder sb); protected abstract Table getTableWithHeader(RestRequest request); @@ -85,6 +90,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes())); }; } else { + if (isActionPaginated()) { + this.paginationQueryMetadata = validateAndGetPaginationMetadata(request); + assert Objects.nonNull(paginationQueryMetadata) : "paginationQueryMetadata can not be null for paginated queries"; + } return doCatRequest(request, client); } } @@ -98,4 +107,47 @@ protected Set responseParams() { return RESPONSE_PARAMS; } + /** + * + * @return boolean denoting whether the RestAction will output paginated responses or not. + * Is kept false by default, every paginated action to override and return true. + */ + public boolean isActionPaginated() { + return false; + } + + /** + * + * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide + * its own implementation. Query params supported by the action specific to pagination along with the respective validations, + * should be added here. The actions would also use the {@param restRequest} to initialise a {@link PageToken}. + */ + protected PaginationQueryMetadata validateAndGetPaginationMetadata(RestRequest restRequest) { + return null; + } + + /** + * A pagination helper class which would contain requested page token and + * a map of query params required by a paginated API. + * + * @opensearch.internal + */ + public static class PaginationQueryMetadata { + private final Map paginationQueryParams; + private final PageToken requestedPageToken; + + public PaginationQueryMetadata(final Map paginationQueryParams, PageToken requestedPageToken) { + this.paginationQueryParams = paginationQueryParams; + this.requestedPageToken = requestedPageToken; + } + + public Map getPaginationQueryParams() { + return paginationQueryParams; + } + + public PageToken getRequestedPageToken() { + return requestedPageToken; + } + } + } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java index 4600dddbb361d..a1cf1ca57c04f 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java @@ -89,7 +89,7 @@ public RestResponse buildResponse(GetAliasesResponse response) throws Exception } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/aliases\n"); sb.append("/_cat/aliases/{alias}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java index 07b0fbbe4a911..2aa2b0aaad893 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java @@ -78,7 +78,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/allocation\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java index 26efd9929afea..9e9b8c2e36b5b 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java @@ -77,7 +77,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/recovery\n"); sb.append("/_cat/recovery/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index aa325443ba6c9..2f76a68e7221e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -58,7 +58,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/segment_replication\n"); sb.append("/_cat/segment_replication/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java index 8f7f9e5bd20a7..3d6211ae50f1a 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java @@ -69,7 +69,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/cluster_manager\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java index 9c054ffe1bcc7..781cfa425f369 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java @@ -71,7 +71,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/count\n"); sb.append("/_cat/count/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java index 04bbdeeadc4c4..cd54b169230d0 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java @@ -82,7 +82,7 @@ public RestResponse buildResponse(NodesStatsResponse nodeStatses) throws Excepti } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/fielddata\n"); sb.append("/_cat/fielddata/{fields}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java index b4d336f4c10c0..898c3a05cec34 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java @@ -69,7 +69,7 @@ public boolean allowSystemIndexAccessByDefault() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/health\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 945834107f904..e2212ca2cdd30 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -61,6 +61,7 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; +import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; import java.time.Instant; import java.time.ZoneOffset; @@ -89,6 +90,7 @@ */ public class RestIndicesAction extends AbstractCatAction { + private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class); private static final String MASTER_TIMEOUT_DEPRECATED_MESSAGE = "Parameter [master_timeout] is deprecated and will be removed in 3.0. To support inclusive language, please use [cluster_manager_timeout] instead."; @@ -111,7 +113,7 @@ public boolean allowSystemIndexAccessByDefault() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/indices\n"); sb.append("/_cat/indices/{index}\n"); } @@ -140,22 +142,29 @@ public RestResponse buildResponse(final Table table) throws Exception { return RestTable.buildResponse(table, channel); } }); - - RestIndicesActionCommonUtils.sendGetSettingsRequest( + sendClusterStateRequest( indices, indicesOptions, local, clusterManagerNodeTimeout, client, - new ActionListener() { + new ActionListener() { @Override - public void onResponse(final GetSettingsResponse getSettingsResponse) { - final GroupedActionListener groupedListener = RestIndicesActionCommonUtils.createGroupedListener( + public void onResponse(final ClusterStateResponse clusterStateResponse) { + IndexBasedPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); + Table.PaginationMetadata paginationMetadata = getTablePaginationMetadata(paginationStrategy); + final String[] indicesToBeQueried = isActionPaginated() + ? paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]) + : indices; + + final GroupedActionListener groupedListener = createGroupedListener( request, 4, - listener + listener, + indicesToBeQueried, + paginationMetadata ); - groupedListener.onResponse(getSettingsResponse); + groupedListener.onResponse(clusterStateResponse); // The list of indices that will be returned is determined by the indices returned from the Get Settings call. // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the @@ -174,23 +183,23 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve // the // index names with the same indices options that we used for the initial cluster state request (strictExpand). - RestIndicesActionCommonUtils.sendIndicesStatsRequest( - indices, + sendGetSettingsRequest( + indicesToBeQueried, subRequestIndicesOptions, - includeUnloadedSegments, + local, + clusterManagerNodeTimeout, client, ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) ); - RestIndicesActionCommonUtils.sendClusterStateRequest( - indices, + sendIndicesStatsRequest( + indicesToBeQueried, subRequestIndicesOptions, - local, - clusterManagerNodeTimeout, + includeUnloadedSegments, client, ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) ); - RestIndicesActionCommonUtils.sendClusterHealthRequest( - indices, + sendClusterHealthRequest( + indicesToBeQueried, subRequestIndicesOptions, local, clusterManagerNodeTimeout, @@ -208,9 +217,134 @@ public void onFailure(final Exception e) { }; } - @Override - protected Table getTableWithHeader(final RestRequest request) { - return RestIndicesActionCommonUtils.getTableWithHeader(request, null); + /** + * We're using the Get Settings API here to resolve the authorized indices for the user. + * This is because the Cluster State and Cluster Health APIs do not filter output based + * on index privileges, so they can't be used to determine which indices are authorized + * or not. On top of this, the Indices Stats API cannot be used either to resolve indices + * as it does not provide information for all existing indices (for example recovering + * indices or non replicated closed indices are not reported in indices stats response). + */ + private void sendGetSettingsRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + final GetSettingsRequest request = new GetSettingsRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey()); + + client.admin().indices().getSettings(request, listener); + } + + private void sendClusterStateRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + + final ClusterStateRequest request = new ClusterStateRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + + client.admin().cluster().state(request, listener); + } + + private void sendClusterHealthRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean local, + final TimeValue clusterManagerNodeTimeout, + final NodeClient client, + final ActionListener listener + ) { + + final ClusterHealthRequest request = new ClusterHealthRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.local(local); + request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); + + client.admin().cluster().health(request, listener); + } + + private void sendIndicesStatsRequest( + final String[] indices, + final IndicesOptions indicesOptions, + final boolean includeUnloadedSegments, + final NodeClient client, + final ActionListener listener + ) { + + final IndicesStatsRequest request = new IndicesStatsRequest(); + request.indices(indices); + request.indicesOptions(indicesOptions); + request.all(); + request.includeUnloadedSegments(includeUnloadedSegments); + + client.admin().indices().stats(request, listener); + } + + private GroupedActionListener createGroupedListener( + final RestRequest request, + final int size, + final ActionListener
listener, + final String[] indicesToBeQueried, + final Table.PaginationMetadata paginationMetadata + ) { + return new GroupedActionListener<>(new ActionListener>() { + @Override + public void onResponse(final Collection responses) { + try { + GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class); + Map indicesSettings = StreamSupport.stream( + Spliterators.spliterator(settingsResponse.getIndexToSettings().entrySet(), 0), + false + ).collect(Collectors.toMap(cursor -> cursor.getKey(), cursor -> cursor.getValue())); + + ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class); + Map indicesStates = StreamSupport.stream( + stateResponse.getState().getMetadata().spliterator(), + false + ).collect(Collectors.toMap(indexMetadata -> indexMetadata.getIndex().getName(), Function.identity())); + + ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class); + Map indicesHealths = healthResponse.getIndices(); + + IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); + Map indicesStats = statsResponse.getIndices(); + + Table responseTable = buildTable( + request, + indicesSettings, + indicesHealths, + indicesStats, + indicesStates, + indicesToBeQueried, + paginationMetadata + ); + listener.onResponse(responseTable); + } catch (Exception e) { + onFailure(e); + } + } + + @Override + public void onFailure(final Exception e) { + listener.onFailure(e); + } + }, size); } private static final Set RESPONSE_PARAMS; @@ -226,815 +360,678 @@ protected Set responseParams() { return RESPONSE_PARAMS; } - public static class RestIndicesActionCommonUtils { - - private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); - - /** - * We're using the Get Settings API here to resolve the authorized indices for the user. - * This is because the Cluster State and Cluster Health APIs do not filter output based - * on index privileges, so they can't be used to determine which indices are authorized - * or not. On top of this, the Indices Stats API cannot be used either to resolve indices - * as it does not provide information for all existing indices (for example recovering - * indices or non replicated closed indices are not reported in indices stats response). - */ - public static void sendGetSettingsRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - final GetSettingsRequest request = new GetSettingsRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - request.names(IndexSettings.INDEX_SEARCH_THROTTLED.getKey()); - - client.admin().indices().getSettings(request, listener); - } - - public static void sendClusterStateRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - - final ClusterStateRequest request = new ClusterStateRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - - client.admin().cluster().state(request, listener); - } - - public static void sendClusterHealthRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean local, - final TimeValue clusterManagerNodeTimeout, - final NodeClient client, - final ActionListener listener - ) { - - final ClusterHealthRequest request = new ClusterHealthRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.local(local); - request.clusterManagerNodeTimeout(clusterManagerNodeTimeout); - - client.admin().cluster().health(request, listener); - } - - public static void sendIndicesStatsRequest( - final String[] indices, - final IndicesOptions indicesOptions, - final boolean includeUnloadedSegments, - final NodeClient client, - final ActionListener listener - ) { - - final IndicesStatsRequest request = new IndicesStatsRequest(); - request.indices(indices); - request.indicesOptions(indicesOptions); - request.all(); - request.includeUnloadedSegments(includeUnloadedSegments); - - client.admin().indices().stats(request, listener); - } - - public static GroupedActionListener createGroupedListener( - final RestRequest request, - final int size, - final ActionListener
listener - ) { - return createGroupedListener(request, size, listener, null); - } - - public static GroupedActionListener createGroupedListener( - final RestRequest request, - final int size, - final ActionListener
listener, - final Table.PaginationMetadata paginationMetadata - ) { - return new GroupedActionListener<>(new ActionListener>() { - @Override - public void onResponse(final Collection responses) { - try { - GetSettingsResponse settingsResponse = extractResponse(responses, GetSettingsResponse.class); - Map indicesSettings = StreamSupport.stream( - Spliterators.spliterator(settingsResponse.getIndexToSettings().entrySet(), 0), - false - ).collect(Collectors.toMap(cursor -> cursor.getKey(), cursor -> cursor.getValue())); - - ClusterStateResponse stateResponse = extractResponse(responses, ClusterStateResponse.class); - Map indicesStates = StreamSupport.stream( - stateResponse.getState().getMetadata().spliterator(), - false - ).collect(Collectors.toMap(indexMetadata -> indexMetadata.getIndex().getName(), Function.identity())); - - ClusterHealthResponse healthResponse = extractResponse(responses, ClusterHealthResponse.class); - Map indicesHealths = healthResponse.getIndices(); - - IndicesStatsResponse statsResponse = extractResponse(responses, IndicesStatsResponse.class); - Map indicesStats = statsResponse.getIndices(); - - Table responseTable = RestIndicesAction.RestIndicesActionCommonUtils.buildTable( - request, - indicesSettings, - indicesHealths, - indicesStats, - indicesStates, - paginationMetadata - ); - listener.onResponse(responseTable); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - listener.onFailure(e); - } - }, size); - } - - public static Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { - Table table = new Table(paginationMetadata); - table.startHeaders(); - table.addCell("health", "alias:h;desc:current health status"); - table.addCell("status", "alias:s;desc:open/close status"); - table.addCell("index", "alias:i,idx;desc:index name"); - table.addCell("uuid", "alias:id,uuid;desc:index uuid"); - table.addCell("pri", "alias:p,shards.primary,shardsPrimary;text-align:right;desc:number of primary shards"); - table.addCell("rep", "alias:r,shards.replica,shardsReplica;text-align:right;desc:number of replica shards"); - table.addCell("docs.count", "alias:dc,docsCount;text-align:right;desc:available docs"); - table.addCell("docs.deleted", "alias:dd,docsDeleted;text-align:right;desc:deleted docs"); - - table.addCell("creation.date", "alias:cd;default:false;desc:index creation date (millisecond value)"); - table.addCell("creation.date.string", "alias:cds;default:false;desc:index creation date (as string)"); - - table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); - table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); - - table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); - table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); - - table.addCell( - "fielddata.memory_size", - "sibling:pri;alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache" - ); - table.addCell("pri.fielddata.memory_size", "default:false;text-align:right;desc:used fielddata cache"); - - table.addCell( - "fielddata.evictions", - "sibling:pri;alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions" - ); - table.addCell("pri.fielddata.evictions", "default:false;text-align:right;desc:fielddata evictions"); - - table.addCell( - "query_cache.memory_size", - "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache" - ); - table.addCell("pri.query_cache.memory_size", "default:false;text-align:right;desc:used query cache"); - - table.addCell( - "query_cache.evictions", - "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions" - ); - table.addCell("pri.query_cache.evictions", "default:false;text-align:right;desc:query cache evictions"); - - table.addCell( - "request_cache.memory_size", - "sibling:pri;alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache" - ); - table.addCell("pri.request_cache.memory_size", "default:false;text-align:right;desc:used request cache"); - - table.addCell( - "request_cache.evictions", - "sibling:pri;alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions" - ); - table.addCell("pri.request_cache.evictions", "default:false;text-align:right;desc:request cache evictions"); - - table.addCell( - "request_cache.hit_count", - "sibling:pri;alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit count" - ); - table.addCell("pri.request_cache.hit_count", "default:false;text-align:right;desc:request cache hit count"); - - table.addCell( - "request_cache.miss_count", - "sibling:pri;alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss count" - ); - table.addCell("pri.request_cache.miss_count", "default:false;text-align:right;desc:request cache miss count"); - - table.addCell("flush.total", "sibling:pri;alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes"); - table.addCell("pri.flush.total", "default:false;text-align:right;desc:number of flushes"); - - table.addCell( - "flush.total_time", - "sibling:pri;alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush" - ); - table.addCell("pri.flush.total_time", "default:false;text-align:right;desc:time spent in flush"); - - table.addCell("get.current", "sibling:pri;alias:gc,getCurrent;default:false;text-align:right;desc:number of current get ops"); - table.addCell("pri.get.current", "default:false;text-align:right;desc:number of current get ops"); - - table.addCell("get.time", "sibling:pri;alias:gti,getTime;default:false;text-align:right;desc:time spent in get"); - table.addCell("pri.get.time", "default:false;text-align:right;desc:time spent in get"); - - table.addCell("get.total", "sibling:pri;alias:gto,getTotal;default:false;text-align:right;desc:number of get ops"); - table.addCell("pri.get.total", "default:false;text-align:right;desc:number of get ops"); - - table.addCell( - "get.exists_time", - "sibling:pri;alias:geti,getExistsTime;default:false;text-align:right;desc:time spent in successful gets" - ); - table.addCell("pri.get.exists_time", "default:false;text-align:right;desc:time spent in successful gets"); - - table.addCell( - "get.exists_total", - "sibling:pri;alias:geto,getExistsTotal;default:false;text-align:right;desc:number of successful gets" - ); - table.addCell("pri.get.exists_total", "default:false;text-align:right;desc:number of successful gets"); - - table.addCell( - "get.missing_time", - "sibling:pri;alias:gmti,getMissingTime;default:false;text-align:right;desc:time spent in failed gets" - ); - table.addCell("pri.get.missing_time", "default:false;text-align:right;desc:time spent in failed gets"); - - table.addCell( - "get.missing_total", - "sibling:pri;alias:gmto,getMissingTotal;default:false;text-align:right;desc:number of failed gets" - ); - table.addCell("pri.get.missing_total", "default:false;text-align:right;desc:number of failed gets"); - - table.addCell( - "indexing.delete_current", - "sibling:pri;alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions" - ); - table.addCell("pri.indexing.delete_current", "default:false;text-align:right;desc:number of current deletions"); - - table.addCell( - "indexing.delete_time", - "sibling:pri;alias:idti,indexingDeleteTime;default:false;text-align:right;desc:time spent in deletions" - ); - table.addCell("pri.indexing.delete_time", "default:false;text-align:right;desc:time spent in deletions"); - - table.addCell( - "indexing.delete_total", - "sibling:pri;alias:idto,indexingDeleteTotal;default:false;text-align:right;desc:number of delete ops" - ); - table.addCell("pri.indexing.delete_total", "default:false;text-align:right;desc:number of delete ops"); - - table.addCell( - "indexing.index_current", - "sibling:pri;alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops" - ); - table.addCell("pri.indexing.index_current", "default:false;text-align:right;desc:number of current indexing ops"); - - table.addCell( - "indexing.index_time", - "sibling:pri;alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing" - ); - table.addCell("pri.indexing.index_time", "default:false;text-align:right;desc:time spent in indexing"); - - table.addCell( - "indexing.index_total", - "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops" - ); - table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops"); - - table.addCell( - "indexing.index_failed", - "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" - ); - table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops"); - - table.addCell( - "merges.current", - "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges" - ); - table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges"); - - table.addCell( - "merges.current_docs", - "sibling:pri;alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs" - ); - table.addCell("pri.merges.current_docs", "default:false;text-align:right;desc:number of current merging docs"); - - table.addCell( - "merges.current_size", - "sibling:pri;alias:mcs,mergesCurrentSize;default:false;text-align:right;desc:size of current merges" - ); - table.addCell("pri.merges.current_size", "default:false;text-align:right;desc:size of current merges"); - - table.addCell( - "merges.total", - "sibling:pri;alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops" - ); - table.addCell("pri.merges.total", "default:false;text-align:right;desc:number of completed merge ops"); - - table.addCell("merges.total_docs", "sibling:pri;alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged"); - table.addCell("pri.merges.total_docs", "default:false;text-align:right;desc:docs merged"); - - table.addCell("merges.total_size", "sibling:pri;alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); - table.addCell("pri.merges.total_size", "default:false;text-align:right;desc:size merged"); - - table.addCell( - "merges.total_time", - "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges" - ); - table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges"); - - table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); - table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes"); - - table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); - table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes"); - - table.addCell( - "refresh.external_total", - "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total external refreshes" - ); - table.addCell("pri.refresh.external_total", "default:false;text-align:right;desc:total external refreshes"); - - table.addCell( - "refresh.external_time", - "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in external refreshes" - ); - table.addCell("pri.refresh.external_time", "default:false;text-align:right;desc:time spent in external refreshes"); - - table.addCell( - "refresh.listeners", - "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners" - ); - table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners"); - - table.addCell( - "search.fetch_current", - "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops" - ); - table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops"); - - table.addCell( - "search.fetch_time", - "sibling:pri;alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase" - ); - table.addCell("pri.search.fetch_time", "default:false;text-align:right;desc:time spent in fetch phase"); - - table.addCell( - "search.fetch_total", - "sibling:pri;alias:sfto,searchFetchTotal;default:false;text-align:right;desc:total fetch ops" - ); - table.addCell("pri.search.fetch_total", "default:false;text-align:right;desc:total fetch ops"); - - table.addCell( - "search.open_contexts", - "sibling:pri;alias:so,searchOpenContexts;default:false;text-align:right;desc:open search contexts" - ); - table.addCell("pri.search.open_contexts", "default:false;text-align:right;desc:open search contexts"); - - table.addCell( - "search.query_current", - "sibling:pri;alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops" - ); - table.addCell("pri.search.query_current", "default:false;text-align:right;desc:current query phase ops"); - - table.addCell( - "search.query_time", - "sibling:pri;alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase" - ); - table.addCell("pri.search.query_time", "default:false;text-align:right;desc:time spent in query phase"); - - table.addCell( - "search.query_total", - "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops" - ); - table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops"); - table.addCell( - "search.concurrent_query_current", - "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" - ); - table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); - - table.addCell( - "search.concurrent_query_time", - "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" - ); - table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); - - table.addCell( - "search.concurrent_query_total", - "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" - ); - table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); - - table.addCell( - "search.concurrent_avg_slice_count", - "sibling:pri;alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" - ); - table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); - - table.addCell( - "search.scroll_current", - "sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts" - ); - table.addCell("pri.search.scroll_current", "default:false;text-align:right;desc:open scroll contexts"); - - table.addCell( - "search.scroll_time", - "sibling:pri;alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open" - ); - table.addCell("pri.search.scroll_time", "default:false;text-align:right;desc:time scroll contexts held open"); - - table.addCell( - "search.scroll_total", - "sibling:pri;alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts" - ); - table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts"); - - table.addCell( - "search.point_in_time_current", - "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts" - ); - table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open point in time contexts"); - - table.addCell( - "search.point_in_time_time", - "sibling:pri;alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open" - ); - table.addCell("pri.search.point_in_time_time", "default:false;text-align:right;desc:time point in time contexts held open"); - - table.addCell( - "search.point_in_time_total", - "sibling:pri;alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts" - ); - table.addCell("pri.search.point_in_time_total", "default:false;text-align:right;desc:completed point in time contexts"); - - table.addCell("segments.count", "sibling:pri;alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); - table.addCell("pri.segments.count", "default:false;text-align:right;desc:number of segments"); - - table.addCell( - "segments.memory", - "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments" - ); - table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments"); - - table.addCell( - "segments.index_writer_memory", - "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer" - ); - table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer"); - - table.addCell( - "segments.version_map_memory", - "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map" - ); - table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map"); - - table.addCell( - "segments.fixed_bitset_memory", - "sibling:pri;alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for" - + " nested object field types and type filters for types referred in _parent fields" - ); - table.addCell( - "pri.segments.fixed_bitset_memory", - "default:false;text-align:right;desc:memory used by fixed bit sets for nested object" - + " field types and type filters for types referred in _parent fields" - ); - - table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops"); - table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops"); - - table.addCell("warmer.total", "sibling:pri;alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops"); - table.addCell("pri.warmer.total", "default:false;text-align:right;desc:total warmer ops"); - - table.addCell( - "warmer.total_time", - "sibling:pri;alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers" - ); - table.addCell("pri.warmer.total_time", "default:false;text-align:right;desc:time spent in warmers"); - - table.addCell( - "suggest.current", - "sibling:pri;alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops" - ); - table.addCell("pri.suggest.current", "default:false;text-align:right;desc:number of current suggest ops"); - - table.addCell("suggest.time", "sibling:pri;alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest"); - table.addCell("pri.suggest.time", "default:false;text-align:right;desc:time spend in suggest"); - - table.addCell("suggest.total", "sibling:pri;alias:suto,suggestTotal;default:false;text-align:right;desc:number of suggest ops"); - table.addCell("pri.suggest.total", "default:false;text-align:right;desc:number of suggest ops"); - - table.addCell("memory.total", "sibling:pri;alias:tm,memoryTotal;default:false;text-align:right;desc:total used memory"); - table.addCell("pri.memory.total", "default:false;text-align:right;desc:total user memory"); + @Override + protected Table getTableWithHeader(final RestRequest request) { + return getTableWithHeader(request, null); + } - table.addCell("search.throttled", "alias:sth;default:false;desc:indicates if the index is search throttled"); + protected Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { + Table table = new Table(paginationMetadata); + table.startHeaders(); + table.addCell("health", "alias:h;desc:current health status"); + table.addCell("status", "alias:s;desc:open/close status"); + table.addCell("index", "alias:i,idx;desc:index name"); + table.addCell("uuid", "alias:id,uuid;desc:index uuid"); + table.addCell("pri", "alias:p,shards.primary,shardsPrimary;text-align:right;desc:number of primary shards"); + table.addCell("rep", "alias:r,shards.replica,shardsReplica;text-align:right;desc:number of replica shards"); + table.addCell("docs.count", "alias:dc,docsCount;text-align:right;desc:available docs"); + table.addCell("docs.deleted", "alias:dd,docsDeleted;text-align:right;desc:deleted docs"); + + table.addCell("creation.date", "alias:cd;default:false;desc:index creation date (millisecond value)"); + table.addCell("creation.date.string", "alias:cds;default:false;desc:index creation date (as string)"); + + table.addCell("store.size", "sibling:pri;alias:ss,storeSize;text-align:right;desc:store size of primaries & replicas"); + table.addCell("pri.store.size", "text-align:right;desc:store size of primaries"); + + table.addCell("completion.size", "sibling:pri;alias:cs,completionSize;default:false;text-align:right;desc:size of completion"); + table.addCell("pri.completion.size", "default:false;text-align:right;desc:size of completion"); + + table.addCell( + "fielddata.memory_size", + "sibling:pri;alias:fm,fielddataMemory;default:false;text-align:right;desc:used fielddata cache" + ); + table.addCell("pri.fielddata.memory_size", "default:false;text-align:right;desc:used fielddata cache"); + + table.addCell( + "fielddata.evictions", + "sibling:pri;alias:fe,fielddataEvictions;default:false;text-align:right;desc:fielddata evictions" + ); + table.addCell("pri.fielddata.evictions", "default:false;text-align:right;desc:fielddata evictions"); + + table.addCell( + "query_cache.memory_size", + "sibling:pri;alias:qcm,queryCacheMemory;default:false;text-align:right;desc:used query cache" + ); + table.addCell("pri.query_cache.memory_size", "default:false;text-align:right;desc:used query cache"); + + table.addCell( + "query_cache.evictions", + "sibling:pri;alias:qce,queryCacheEvictions;default:false;text-align:right;desc:query cache evictions" + ); + table.addCell("pri.query_cache.evictions", "default:false;text-align:right;desc:query cache evictions"); + + table.addCell( + "request_cache.memory_size", + "sibling:pri;alias:rcm,requestCacheMemory;default:false;text-align:right;desc:used request cache" + ); + table.addCell("pri.request_cache.memory_size", "default:false;text-align:right;desc:used request cache"); + + table.addCell( + "request_cache.evictions", + "sibling:pri;alias:rce,requestCacheEvictions;default:false;text-align:right;desc:request cache evictions" + ); + table.addCell("pri.request_cache.evictions", "default:false;text-align:right;desc:request cache evictions"); + + table.addCell( + "request_cache.hit_count", + "sibling:pri;alias:rchc,requestCacheHitCount;default:false;text-align:right;desc:request cache hit count" + ); + table.addCell("pri.request_cache.hit_count", "default:false;text-align:right;desc:request cache hit count"); + + table.addCell( + "request_cache.miss_count", + "sibling:pri;alias:rcmc,requestCacheMissCount;default:false;text-align:right;desc:request cache miss count" + ); + table.addCell("pri.request_cache.miss_count", "default:false;text-align:right;desc:request cache miss count"); + + table.addCell("flush.total", "sibling:pri;alias:ft,flushTotal;default:false;text-align:right;desc:number of flushes"); + table.addCell("pri.flush.total", "default:false;text-align:right;desc:number of flushes"); + + table.addCell("flush.total_time", "sibling:pri;alias:ftt,flushTotalTime;default:false;text-align:right;desc:time spent in flush"); + table.addCell("pri.flush.total_time", "default:false;text-align:right;desc:time spent in flush"); + + table.addCell("get.current", "sibling:pri;alias:gc,getCurrent;default:false;text-align:right;desc:number of current get ops"); + table.addCell("pri.get.current", "default:false;text-align:right;desc:number of current get ops"); + + table.addCell("get.time", "sibling:pri;alias:gti,getTime;default:false;text-align:right;desc:time spent in get"); + table.addCell("pri.get.time", "default:false;text-align:right;desc:time spent in get"); + + table.addCell("get.total", "sibling:pri;alias:gto,getTotal;default:false;text-align:right;desc:number of get ops"); + table.addCell("pri.get.total", "default:false;text-align:right;desc:number of get ops"); + + table.addCell( + "get.exists_time", + "sibling:pri;alias:geti,getExistsTime;default:false;text-align:right;desc:time spent in successful gets" + ); + table.addCell("pri.get.exists_time", "default:false;text-align:right;desc:time spent in successful gets"); + + table.addCell( + "get.exists_total", + "sibling:pri;alias:geto,getExistsTotal;default:false;text-align:right;desc:number of successful gets" + ); + table.addCell("pri.get.exists_total", "default:false;text-align:right;desc:number of successful gets"); + + table.addCell( + "get.missing_time", + "sibling:pri;alias:gmti,getMissingTime;default:false;text-align:right;desc:time spent in failed gets" + ); + table.addCell("pri.get.missing_time", "default:false;text-align:right;desc:time spent in failed gets"); + + table.addCell( + "get.missing_total", + "sibling:pri;alias:gmto,getMissingTotal;default:false;text-align:right;desc:number of failed gets" + ); + table.addCell("pri.get.missing_total", "default:false;text-align:right;desc:number of failed gets"); + + table.addCell( + "indexing.delete_current", + "sibling:pri;alias:idc,indexingDeleteCurrent;default:false;text-align:right;desc:number of current deletions" + ); + table.addCell("pri.indexing.delete_current", "default:false;text-align:right;desc:number of current deletions"); + + table.addCell( + "indexing.delete_time", + "sibling:pri;alias:idti,indexingDeleteTime;default:false;text-align:right;desc:time spent in deletions" + ); + table.addCell("pri.indexing.delete_time", "default:false;text-align:right;desc:time spent in deletions"); + + table.addCell( + "indexing.delete_total", + "sibling:pri;alias:idto,indexingDeleteTotal;default:false;text-align:right;desc:number of delete ops" + ); + table.addCell("pri.indexing.delete_total", "default:false;text-align:right;desc:number of delete ops"); + + table.addCell( + "indexing.index_current", + "sibling:pri;alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops" + ); + table.addCell("pri.indexing.index_current", "default:false;text-align:right;desc:number of current indexing ops"); + + table.addCell( + "indexing.index_time", + "sibling:pri;alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing" + ); + table.addCell("pri.indexing.index_time", "default:false;text-align:right;desc:time spent in indexing"); + + table.addCell( + "indexing.index_total", + "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops" + ); + table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops"); + + table.addCell( + "indexing.index_failed", + "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops" + ); + table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops"); + + table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges"); + table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges"); + + table.addCell( + "merges.current_docs", + "sibling:pri;alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs" + ); + table.addCell("pri.merges.current_docs", "default:false;text-align:right;desc:number of current merging docs"); + + table.addCell( + "merges.current_size", + "sibling:pri;alias:mcs,mergesCurrentSize;default:false;text-align:right;desc:size of current merges" + ); + table.addCell("pri.merges.current_size", "default:false;text-align:right;desc:size of current merges"); + + table.addCell("merges.total", "sibling:pri;alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops"); + table.addCell("pri.merges.total", "default:false;text-align:right;desc:number of completed merge ops"); + + table.addCell("merges.total_docs", "sibling:pri;alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged"); + table.addCell("pri.merges.total_docs", "default:false;text-align:right;desc:docs merged"); + + table.addCell("merges.total_size", "sibling:pri;alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged"); + table.addCell("pri.merges.total_size", "default:false;text-align:right;desc:size merged"); + + table.addCell( + "merges.total_time", + "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges" + ); + table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges"); + + table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes"); + table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes"); + + table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes"); + table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes"); + + table.addCell( + "refresh.external_total", + "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total external refreshes" + ); + table.addCell("pri.refresh.external_total", "default:false;text-align:right;desc:total external refreshes"); + + table.addCell( + "refresh.external_time", + "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in external refreshes" + ); + table.addCell("pri.refresh.external_time", "default:false;text-align:right;desc:time spent in external refreshes"); + + table.addCell( + "refresh.listeners", + "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners" + ); + table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners"); + + table.addCell( + "search.fetch_current", + "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops" + ); + table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops"); + + table.addCell( + "search.fetch_time", + "sibling:pri;alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase" + ); + table.addCell("pri.search.fetch_time", "default:false;text-align:right;desc:time spent in fetch phase"); + + table.addCell("search.fetch_total", "sibling:pri;alias:sfto,searchFetchTotal;default:false;text-align:right;desc:total fetch ops"); + table.addCell("pri.search.fetch_total", "default:false;text-align:right;desc:total fetch ops"); + + table.addCell( + "search.open_contexts", + "sibling:pri;alias:so,searchOpenContexts;default:false;text-align:right;desc:open search contexts" + ); + table.addCell("pri.search.open_contexts", "default:false;text-align:right;desc:open search contexts"); + + table.addCell( + "search.query_current", + "sibling:pri;alias:sqc,searchQueryCurrent;default:false;text-align:right;desc:current query phase ops" + ); + table.addCell("pri.search.query_current", "default:false;text-align:right;desc:current query phase ops"); + + table.addCell( + "search.query_time", + "sibling:pri;alias:sqti,searchQueryTime;default:false;text-align:right;desc:time spent in query phase" + ); + table.addCell("pri.search.query_time", "default:false;text-align:right;desc:time spent in query phase"); + + table.addCell( + "search.query_total", + "sibling:pri;alias:sqto,searchQueryTotal;default:false;text-align:right;desc:total query phase ops" + ); + table.addCell("pri.search.query_total", "default:false;text-align:right;desc:total query phase ops"); + table.addCell( + "search.concurrent_query_current", + "sibling:pri;alias:scqc,searchConcurrentQueryCurrent;default:false;text-align:right;desc:current concurrent query phase ops" + ); + table.addCell("pri.search.concurrent_query_current", "default:false;text-align:right;desc:current concurrent query phase ops"); + + table.addCell( + "search.concurrent_query_time", + "sibling:pri;alias:scqti,searchConcurrentQueryTime;default:false;text-align:right;desc:time spent in concurrent query phase" + ); + table.addCell("pri.search.concurrent_query_time", "default:false;text-align:right;desc:time spent in concurrent query phase"); + + table.addCell( + "search.concurrent_query_total", + "sibling:pri;alias:scqto,searchConcurrentQueryTotal;default:false;text-align:right;desc:total query phase ops" + ); + table.addCell("pri.search.concurrent_query_total", "default:false;text-align:right;desc:total query phase ops"); + + table.addCell( + "search.concurrent_avg_slice_count", + "sibling:pri;alias:casc,searchConcurrentAvgSliceCount;default:false;text-align:right;desc:average query concurrency" + ); + table.addCell("pri.search.concurrent_avg_slice_count", "default:false;text-align:right;desc:average query concurrency"); + + table.addCell( + "search.scroll_current", + "sibling:pri;alias:scc,searchScrollCurrent;default:false;text-align:right;desc:open scroll contexts" + ); + table.addCell("pri.search.scroll_current", "default:false;text-align:right;desc:open scroll contexts"); + + table.addCell( + "search.scroll_time", + "sibling:pri;alias:scti,searchScrollTime;default:false;text-align:right;desc:time scroll contexts held open" + ); + table.addCell("pri.search.scroll_time", "default:false;text-align:right;desc:time scroll contexts held open"); + + table.addCell( + "search.scroll_total", + "sibling:pri;alias:scto,searchScrollTotal;default:false;text-align:right;desc:completed scroll contexts" + ); + table.addCell("pri.search.scroll_total", "default:false;text-align:right;desc:completed scroll contexts"); + + table.addCell( + "search.point_in_time_current", + "sibling:pri;alias:scc,searchPointInTimeCurrent;default:false;text-align:right;desc:open point in time contexts" + ); + table.addCell("pri.search.point_in_time_current", "default:false;text-align:right;desc:open point in time contexts"); + + table.addCell( + "search.point_in_time_time", + "sibling:pri;alias:scti,searchPointInTimeTime;default:false;text-align:right;desc:time point in time contexts held open" + ); + table.addCell("pri.search.point_in_time_time", "default:false;text-align:right;desc:time point in time contexts held open"); + + table.addCell( + "search.point_in_time_total", + "sibling:pri;alias:scto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts" + ); + table.addCell("pri.search.point_in_time_total", "default:false;text-align:right;desc:completed point in time contexts"); + + table.addCell("segments.count", "sibling:pri;alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments"); + table.addCell("pri.segments.count", "default:false;text-align:right;desc:number of segments"); + + table.addCell("segments.memory", "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments"); + table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments"); + + table.addCell( + "segments.index_writer_memory", + "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer" + ); + table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer"); + + table.addCell( + "segments.version_map_memory", + "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map" + ); + table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map"); + + table.addCell( + "segments.fixed_bitset_memory", + "sibling:pri;alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for" + + " nested object field types and type filters for types referred in _parent fields" + ); + table.addCell( + "pri.segments.fixed_bitset_memory", + "default:false;text-align:right;desc:memory used by fixed bit sets for nested object" + + " field types and type filters for types referred in _parent fields" + ); + + table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops"); + table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops"); + + table.addCell("warmer.total", "sibling:pri;alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops"); + table.addCell("pri.warmer.total", "default:false;text-align:right;desc:total warmer ops"); + + table.addCell( + "warmer.total_time", + "sibling:pri;alias:wtt,warmerTotalTime;default:false;text-align:right;desc:time spent in warmers" + ); + table.addCell("pri.warmer.total_time", "default:false;text-align:right;desc:time spent in warmers"); + + table.addCell( + "suggest.current", + "sibling:pri;alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops" + ); + table.addCell("pri.suggest.current", "default:false;text-align:right;desc:number of current suggest ops"); + + table.addCell("suggest.time", "sibling:pri;alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest"); + table.addCell("pri.suggest.time", "default:false;text-align:right;desc:time spend in suggest"); + + table.addCell("suggest.total", "sibling:pri;alias:suto,suggestTotal;default:false;text-align:right;desc:number of suggest ops"); + table.addCell("pri.suggest.total", "default:false;text-align:right;desc:number of suggest ops"); + + table.addCell("memory.total", "sibling:pri;alias:tm,memoryTotal;default:false;text-align:right;desc:total used memory"); + table.addCell("pri.memory.total", "default:false;text-align:right;desc:total user memory"); + + table.addCell("search.throttled", "alias:sth;default:false;desc:indicates if the index is search throttled"); + + table.endHeaders(); + return table; + } - table.endHeaders(); + // package private for testing + Table buildTable( + final RestRequest request, + final Map indicesSettings, + final Map indicesHealths, + final Map indicesStats, + final Map indicesMetadatas, + final String[] indicesToBeQueried, + final Table.PaginationMetadata paginationMetadata + ) { + + final String healthParam = request.param("health"); + final Table table = getTableWithHeader(request, paginationMetadata); + + if (isActionPaginated() && indicesToBeQueried.length == 0) { + // to handle cases where paginationStrategy couldn't find any indices that should be queried + // and would have returned an empty list of indicesToBeQueried. return table; } - // package private for testing - public static Table buildTable( - final RestRequest request, - final Map indicesSettings, - final Map indicesHealths, - final Map indicesStats, - final Map indicesMetadatas, - final Table.PaginationMetadata paginationMetadata - ) { - - final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request, paginationMetadata); - - indicesSettings.forEach((indexName, settings) -> { - if (indicesMetadatas.containsKey(indexName) == false) { - // the index exists in the Get Indices response but is not present in the cluster state: - // it is likely that the index was deleted in the meanwhile, so we ignore it. - return; - } + indicesSettings.forEach((indexName, settings) -> { + if (indicesMetadatas.containsKey(indexName) == false) { + // the index exists in the Get Indices response but is not present in the cluster state: + // it is likely that the index was deleted in the meanwhile, so we ignore it. + return; + } - final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); - final IndexMetadata.State indexState = indexMetadata.getState(); - final IndexStats indexStats = indicesStats.get(indexName); - final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); + final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); + final IndexMetadata.State indexState = indexMetadata.getState(); + final IndexStats indexStats = indicesStats.get(indexName); + final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); + + final String health; + final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); + if (indexHealth != null) { + health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); + } else if (indexStats != null) { + health = "red*"; + } else { + health = ""; + } - final String health; - final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); + if (healthParam != null) { + final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); + boolean skip; if (indexHealth != null) { - health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); - } else if (indexStats != null) { - health = "red*"; + // index health is known but does not match the one requested + skip = indexHealth.getStatus() != healthStatusFilter; } else { - health = ""; + // index health is unknown, skip if we don't explicitly request RED health + skip = ClusterHealthStatus.RED != healthStatusFilter; } - - if (healthParam != null) { - final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); - boolean skip; - if (indexHealth != null) { - // index health is known but does not match the one requested - skip = indexHealth.getStatus() != healthStatusFilter; - } else { - // index health is unknown, skip if we don't explicitly request RED health - skip = ClusterHealthStatus.RED != healthStatusFilter; - } - if (skip) { - return; - } + if (skip) { + return; } + } - final CommonStats primaryStats; - final CommonStats totalStats; + final CommonStats primaryStats; + final CommonStats totalStats; - if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { - // TODO: expose docs stats for replicated closed indices - primaryStats = new CommonStats(); - totalStats = new CommonStats(); - } else { - primaryStats = indexStats.getPrimaries(); - totalStats = indexStats.getTotal(); - } - table.startRow(); - table.addCell(health); - table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); - table.addCell(indexName); - table.addCell(indexMetadata.getIndexUUID()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); + if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { + // TODO: expose docs stats for replicated closed indices + primaryStats = new CommonStats(); + totalStats = new CommonStats(); + } else { + primaryStats = indexStats.getPrimaries(); + totalStats = indexStats.getTotal(); + } + table.startRow(); + table.addCell(health); + table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); + table.addCell(indexName); + table.addCell(indexMetadata.getIndexUUID()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); - table.addCell(indexMetadata.getCreationDate()); - ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); - table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); + table.addCell(indexMetadata.getCreationDate()); + ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); + table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); - table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); - table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); + table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); - table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); - table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); + table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); - table.addCell(totalStats.getTotalMemory()); - table.addCell(primaryStats.getTotalMemory()); + table.addCell(totalStats.getTotalMemory()); + table.addCell(primaryStats.getTotalMemory()); - table.addCell(searchThrottled); + table.addCell(searchThrottled); - table.endRow(); - }); + table.endRow(); + }); - return table; - } + return table; + } - @SuppressWarnings("unchecked") - private static A extractResponse(final Collection responses, Class c) { - return (A) responses.stream().filter(c::isInstance).findFirst().get(); - } + @SuppressWarnings("unchecked") + private static A extractResponse(final Collection responses, Class c) { + return (A) responses.stream().filter(c::isInstance).findFirst().get(); + } + + protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return null; + } + protected Table.PaginationMetadata getTablePaginationMetadata(IndexBasedPaginationStrategy paginationStrategy) { + return null; } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodeAttrsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodeAttrsAction.java index 5ead69320fefd..37f1805c4fdf3 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodeAttrsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodeAttrsAction.java @@ -75,7 +75,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/nodeattrs\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java index e11012a23fce7..43825707f60b6 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestNodesAction.java @@ -103,7 +103,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/nodes\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestPendingClusterTasksAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestPendingClusterTasksAction.java index 472d04e5a1679..98a0a28bed1f1 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestPendingClusterTasksAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestPendingClusterTasksAction.java @@ -67,7 +67,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/pending_tasks\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java index 5fc6c961b4637..7941d00bcb0df 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestPitSegmentsAction.java @@ -88,7 +88,7 @@ public RestResponse buildResponse(final IndicesSegmentResponse indicesSegmentRes } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/pit_segments\n"); sb.append("/_cat/pit_segments/{pit_id}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestPluginsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestPluginsAction.java index 2f3794cd1b9f9..137ab2dd8a52e 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestPluginsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestPluginsAction.java @@ -74,7 +74,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/plugins\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestRepositoriesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestRepositoriesAction.java index 19079cd9975ba..a8dab7af54a41 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestRepositoriesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestRepositoriesAction.java @@ -86,7 +86,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/repositories\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java index b88af4ac3eeed..742079c126fd5 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSegmentsAction.java @@ -111,7 +111,7 @@ public RestResponse buildResponse(final IndicesSegmentResponse indicesSegmentRes } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/segments\n"); sb.append("/_cat/segments/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java index 4413c8eb370be..29f5b84464790 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java @@ -99,7 +99,7 @@ public boolean allowSystemIndexAccessByDefault() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/shards\n"); sb.append("/_cat/shards/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java index f05a84d9b2aa2..fb07a106528f1 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java @@ -96,7 +96,7 @@ public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) thr } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/snapshots/{repository}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java index 560b88787ae09..d83d2438dbb99 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java @@ -85,7 +85,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/tasks\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java index 0e9ad8760d4b8..fafa15820e030 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java @@ -72,7 +72,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/templates\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java index 0393dd15c8238..2267f9200aece 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java @@ -87,7 +87,7 @@ public String getName() { } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_cat/thread_pool\n"); sb.append("/_cat/thread_pool/{thread_pools}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java deleted file mode 100644 index ff32e3c49434f..0000000000000 --- a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.action.list; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Table; -import org.opensearch.common.io.Streams; -import org.opensearch.common.io.UTF8StreamWriter; -import org.opensearch.core.common.io.stream.BytesStream; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.RestRequest; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import static org.opensearch.rest.action.cat.RestTable.buildHelpWidths; -import static org.opensearch.rest.action.cat.RestTable.pad; - -/** - * Base Transport action class for _list APIs - * - * @opensearch.api - */ -public abstract class AbstractListAction extends BaseRestHandler { - protected abstract RestChannelConsumer doListRequest(RestRequest request, NodeClient client); - - protected abstract void documentation(StringBuilder sb); - - protected abstract Table getTableWithHeader(RestRequest request); - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - boolean helpWanted = request.paramAsBoolean("help", false); - if (helpWanted) { - return channel -> { - Table table = getTableWithHeader(request); - int[] width = buildHelpWidths(table, request); - BytesStream bytesOutput = Streams.flushOnCloseStream(channel.bytesOutput()); - UTF8StreamWriter out = new UTF8StreamWriter().setOutput(bytesOutput); - for (Table.Cell cell : table.getHeaders()) { - // need to do left-align always, so create new cells - pad(new Table.Cell(cell.value), width[0], request, out); - out.append(" | "); - pad(new Table.Cell(cell.attr.containsKey("alias") ? cell.attr.get("alias") : ""), width[1], request, out); - out.append(" | "); - pad(new Table.Cell(cell.attr.containsKey("desc") ? cell.attr.get("desc") : "not available"), width[2], request, out); - out.append("\n"); - } - out.close(); - channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes())); - }; - } else { - return doListRequest(request, client); - } - } - - static Set RESPONSE_PARAMS = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("format", "h", "v", "ts", "pri", "bytes", "size", "time", "s", "timeout")) - ); - - @Override - protected Set responseParams() { - return RESPONSE_PARAMS; - } - -} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index ee1a5fc9f1411..b120f308f3d5e 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,29 +9,18 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.IndicesOptions; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.Table; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.action.ActionResponse; -import org.opensearch.core.common.Strings; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestResponse; -import org.opensearch.rest.action.RestResponseListener; import org.opensearch.rest.action.cat.RestIndicesAction; -import org.opensearch.rest.action.cat.RestTable; import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; -import java.util.Collections; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; -import java.util.Set; +import java.util.Map; +import java.util.Objects; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; -import static org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest.DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT; import static org.opensearch.rest.RestRequest.Method.GET; /** @@ -39,9 +28,11 @@ * * @opensearch.api */ -public class RestIndicesListAction extends AbstractListAction { - private static final String DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = "1000"; - private static final String PAGINATED_ELEMENT_KEY = "indices"; +public class RestIndicesListAction extends RestIndicesAction { + + protected static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING = 1000; + protected static final int DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = 1000; + protected static final String PAGINATED_LIST_INDICES_ELEMENT_KEY = "indices"; @Override public List routes() { @@ -54,122 +45,60 @@ public String getName() { } @Override - public boolean allowSystemIndexAccessByDefault() { - return true; - } - - @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { sb.append("/_list/indices\n"); sb.append("/_list/indices/{index}\n"); } @Override - public RestChannelConsumer doListRequest(final RestRequest request, final NodeClient client) { - final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final boolean local = request.paramAsBoolean("local", false); - TimeValue clusterManagerTimeout = request.paramAsTime("cluster_manager_timeout", DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT); - final boolean includeUnloadedSegments = request.paramAsBoolean("include_unloaded_segments", false); - final String requestedToken = request.param("next_token"); - final int pageSize = Integer.parseInt(request.param("size", DEFAULT_LIST_INDICES_PAGE_SIZE_STRING)); - final String requestedSortOrder = request.param("sort", "ascending"); - - return channel -> { - final ActionListener
listener = ActionListener.notifyOnce(new RestResponseListener
(channel) { - @Override - public RestResponse buildResponse(final Table table) throws Exception { - return RestTable.buildResponse(table, channel); - } - }); - - // Fetch all the indices from clusterStateRequest for a paginated query. - RestIndicesAction.RestIndicesActionCommonUtils.sendClusterStateRequest( - indices, - IndicesOptions.lenientExpandHidden(), - local, - clusterManagerTimeout, - client, - new ActionListener() { - @Override - public void onResponse(final ClusterStateResponse clusterStateResponse) { - try { - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy( - requestedToken == null ? null : new IndexBasedPaginationStrategy.IndexStrategyPageToken(requestedToken), - pageSize, - requestedSortOrder, - clusterStateResponse.getState() - ); - - final GroupedActionListener groupedListener = RestIndicesAction.RestIndicesActionCommonUtils - .createGroupedListener( - request, - 4, - listener, - new Table.PaginationMetadata( - true, - PAGINATED_ELEMENT_KEY, - paginationStrategy.getNextToken() == null - ? null - : paginationStrategy.getNextToken().generateEncryptedToken() - ) - ); - groupedListener.onResponse(clusterStateResponse); - - final String[] indicesToBeQueried = paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]); - RestIndicesAction.RestIndicesActionCommonUtils.sendGetSettingsRequest( - indicesToBeQueried, - IndicesOptions.fromRequest(request, IndicesOptions.strictExpand()), - local, - clusterManagerTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - RestIndicesAction.RestIndicesActionCommonUtils.sendIndicesStatsRequest( - indicesToBeQueried, - IndicesOptions.lenientExpandHidden(), - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - RestIndicesAction.RestIndicesActionCommonUtils.sendClusterHealthRequest( - indicesToBeQueried, - IndicesOptions.lenientExpandHidden(), - local, - clusterManagerTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(final Exception e) { - listener.onFailure(e); - } - } - ); - }; - + public boolean isActionPaginated() { + return true; } - private static final Set RESPONSE_PARAMS; - - static { - final Set responseParams = new HashSet<>(asList("local", "health")); - responseParams.addAll(AbstractListAction.RESPONSE_PARAMS); - RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + @Override + protected PaginationQueryMetadata validateAndGetPaginationMetadata(RestRequest restRequest) { + Map paginatedQueryParams = new HashMap<>(); + final String requestedTokenStr = restRequest.param("next_token"); + final String sortOrder = restRequest.param("sort", "ascending"); + final int pageSize = restRequest.paramAsInt("size", DEFAULT_LIST_INDICES_PAGE_SIZE_STRING); + // validating pageSize + if (pageSize <= 0) { + throw new IllegalArgumentException("size must be greater than zero"); + } else if (pageSize > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { + throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING + "]"); + } + // Validating sort order + if (!Objects.equals(sortOrder, "ascending") && !Objects.equals(sortOrder, "descending")) { + throw new IllegalArgumentException("value of sort can either be ascending or descending"); + } + // Next Token in the request will be validated by the IndexStrategyPageToken itself. + IndexBasedPaginationStrategy.IndexStrategyPageToken requestedPageToken = requestedTokenStr == null + ? null + : new IndexBasedPaginationStrategy.IndexStrategyPageToken(requestedTokenStr); + paginatedQueryParams.put("next_token", requestedTokenStr); + paginatedQueryParams.put("size", pageSize); + paginatedQueryParams.put("sort", sortOrder); + return new PaginationQueryMetadata(paginatedQueryParams, requestedPageToken); } @Override - protected Set responseParams() { - return RESPONSE_PARAMS; + protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + assert !Objects.nonNull(paginationQueryMetadata.getRequestedPageToken()) + || paginationQueryMetadata.getRequestedPageToken() instanceof IndexBasedPaginationStrategy.IndexStrategyPageToken; + return new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationQueryMetadata.getRequestedPageToken(), + (int) paginationQueryMetadata.getPaginationQueryParams().get("size"), + (String) paginationQueryMetadata.getPaginationQueryParams().get("sort"), + clusterStateResponse.getState() + ); } @Override - protected Table getTableWithHeader(final RestRequest request) { - return RestIndicesAction.RestIndicesActionCommonUtils.getTableWithHeader(request, null); + protected Table.PaginationMetadata getTablePaginationMetadata(IndexBasedPaginationStrategy paginationStrategy) { + return new Table.PaginationMetadata( + true, + PAGINATED_LIST_INDICES_ELEMENT_KEY, + paginationStrategy.getNextToken() == null ? null : paginationStrategy.getNextToken().generateEncryptedToken() + ); } - } diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java index 4b8551ea7e14a..32e5824b45cec 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java @@ -13,6 +13,7 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.AbstractCatAction; import java.io.IOException; import java.util.List; @@ -31,10 +32,10 @@ public class RestListAction extends BaseRestHandler { private static final String LIST_NL = LIST + "\n"; private final String HELP; - public RestListAction(List listActions) { + public RestListAction(List listActions) { StringBuilder sb = new StringBuilder(); sb.append(LIST_NL); - for (AbstractListAction listAction : listActions) { + for (AbstractCatAction listAction : listActions) { listAction.documentation(sb); } HELP = sb.toString(); diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java index 25dd50a5ebb39..f79c56e88d994 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -12,12 +12,9 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.common.Nullable; -import java.util.Base64; +import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; - -import static java.nio.charset.StandardCharsets.UTF_8; /** * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. @@ -27,34 +24,52 @@ */ public class IndexBasedPaginationStrategy implements PaginationStrategy { - private static final String DESCENDING_SORT_PARAM_VALUE = "descending"; private final IndexStrategyPageToken nextToken; private final List indicesFromRequestedToken; - private static final String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = - "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; - public IndexBasedPaginationStrategy( @Nullable IndexStrategyPageToken requestedToken, int maxPageSize, String sortOrder, ClusterState clusterState ) { - // Get sorted list of indices from metadata and filter out the required number of indices - List sortedIndicesList = getListOfIndicesSortedByCreateTime(clusterState, sortOrder, requestedToken); - final int newPageStartIndexNumber = getNewPageIndexStartNumber(requestedToken, sortedIndicesList, clusterState); // inclusive - int newPageEndIndexNumber = Math.min(newPageStartIndexNumber + maxPageSize, sortedIndicesList.size()); // exclusive - this.indicesFromRequestedToken = sortedIndicesList.subList(newPageStartIndexNumber, newPageEndIndexNumber); - long queryStartTime = requestedToken == null - ? clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() - : requestedToken.queryStartTime; - IndexStrategyPageToken nextPageToken = new IndexStrategyPageToken( - newPageEndIndexNumber, - clusterState.metadata().indices().get(sortedIndicesList.get(newPageEndIndexNumber - 1)).getCreationDate(), - queryStartTime, - sortedIndicesList.get(newPageEndIndexNumber - 1) + // Get sorted list of indices not containing the ones created after query start time + List sortedIndicesList = PaginationStrategy.getListOfIndicesSortedByCreateTime( + clusterState, + sortOrder, + requestedToken == null ? Long.MAX_VALUE : requestedToken.queryStartTime ); - this.nextToken = newPageEndIndexNumber >= sortedIndicesList.size() ? null : nextPageToken; + if (sortedIndicesList.isEmpty()) { + // Denotes, that all the indices which were created before the queryStartTime have been deleted. + // No nextToken and indices need to be shown in such cases. + this.indicesFromRequestedToken = new ArrayList<>(); + this.nextToken = null; + } else { + final int requestedPageStartIndexNumber = getRequestedPageIndexStartNumber( + requestedToken, + sortedIndicesList, + clusterState, + sortOrder + ); // inclusive + int requestedPageEndIndexNumber = Math.min(requestedPageStartIndexNumber + maxPageSize, sortedIndicesList.size()); // exclusive + + this.indicesFromRequestedToken = sortedIndicesList.subList(requestedPageStartIndexNumber, requestedPageEndIndexNumber); + + // Set the queryStart time as the timestamp of latest created index if requested token is null. + long queryStartTime = requestedToken == null + ? DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + ? clusterState.metadata().indices().get(sortedIndicesList.get(0)).getCreationDate() + : clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() + : requestedToken.queryStartTime; + this.nextToken = requestedPageEndIndexNumber >= sortedIndicesList.size() + ? null + : new IndexStrategyPageToken( + requestedPageEndIndexNumber, + clusterState.metadata().indices().get(sortedIndicesList.get(requestedPageEndIndexNumber - 1)).getCreationDate(), + queryStartTime, + sortedIndicesList.get(requestedPageEndIndexNumber - 1) + ); + } } @Override @@ -64,62 +79,39 @@ public PageToken getNextToken() { } @Override - @Nullable public List getElementsFromRequestedToken() { - return indicesFromRequestedToken; - } - - private List getListOfIndicesSortedByCreateTime( - final ClusterState clusterState, - String sortOrder, - IndexStrategyPageToken requestedPageToken - ) { - long latestValidIndexCreateTime = requestedPageToken == null ? Long.MAX_VALUE : requestedPageToken.queryStartTime; - // Filter out the indices which have been created after the latest index which was present when paginated query started. - // Also, sort the indices list based on their creation timestamps - return clusterState.getRoutingTable() - .getIndicesRouting() - .keySet() - .stream() - .filter(index -> (latestValidIndexCreateTime - clusterState.metadata().indices().get(index).getCreationDate()) >= 0) - .sorted((index1, index2) -> { - Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); - Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); - if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { - return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) ? index2.compareTo(index1) : index1.compareTo(index2); - } - return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) - : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); - }) - .collect(Collectors.toList()); + return Objects.isNull(indicesFromRequestedToken) ? new ArrayList<>() : indicesFromRequestedToken; } - private int getNewPageIndexStartNumber( + private int getRequestedPageIndexStartNumber( final IndexStrategyPageToken requestedPageToken, final List sortedIndicesList, - final ClusterState clusterState + final ClusterState clusterState, + final String sortOrder ) { if (Objects.isNull(requestedPageToken)) { return 0; } - int newPageStartIndexNumber = Math.min(requestedPageToken.posToStartPage, sortedIndicesList.size() - 1); - if (newPageStartIndexNumber > 0 - && !Objects.equals(sortedIndicesList.get(newPageStartIndexNumber - 1), requestedPageToken.nameOfLastRespondedIndex)) { + int requestedPageStartIndexNumber = Math.min(requestedPageToken.posToStartPage, sortedIndicesList.size() - 1); + if (requestedPageStartIndexNumber > 0 + && !Objects.equals(sortedIndicesList.get(requestedPageStartIndexNumber - 1), requestedPageToken.nameOfLastRespondedIndex)) { // case denoting an already responded index has been deleted while the paginated queries are being executed - // find the index whose creation time is just after the index which was last responded - newPageStartIndexNumber--; - while (newPageStartIndexNumber > 0) { - if (clusterState.metadata() + // find the index whose creation time is just after/before (based on sortOrder) the index which was last responded. + while (requestedPageStartIndexNumber > 0) { + long creationTimeOfIndex = clusterState.metadata() .indices() - .get(sortedIndicesList.get(newPageStartIndexNumber - 1)) - .getCreationDate() < requestedPageToken.creationTimeOfLastRespondedIndex) { + .get(sortedIndicesList.get(requestedPageStartIndexNumber - 1)) + .getCreationDate(); + if ((DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + && creationTimeOfIndex < requestedPageToken.creationTimeOfLastRespondedIndex) + || (DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + && creationTimeOfIndex > requestedPageToken.creationTimeOfLastRespondedIndex)) { break; } - newPageStartIndexNumber--; + requestedPageStartIndexNumber--; } } - return newPageStartIndexNumber; + return requestedPageStartIndexNumber; } /** @@ -143,22 +135,18 @@ public static class IndexStrategyPageToken implements PageToken { */ public IndexStrategyPageToken(String requestedTokenString) { Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); - try { - requestedTokenString = new String(Base64.getDecoder().decode(requestedTokenString), UTF_8); - } catch (IllegalArgumentException exception) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } + String decryptedToken = PageToken.decryptStringToken(requestedTokenString); - final String[] requestedTokenElements = requestedTokenString.split("\\$"); - if (requestedTokenElements.length != 4) { + final String[] decryptedTokenElements = decryptedToken.split("\\$"); + if (decryptedTokenElements.length != 4) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } try { - this.posToStartPage = Integer.parseInt(requestedTokenElements[0]); - this.creationTimeOfLastRespondedIndex = Long.parseLong(requestedTokenElements[1]); - this.queryStartTime = Long.parseLong(requestedTokenElements[2]); - this.nameOfLastRespondedIndex = requestedTokenElements[3]; + this.posToStartPage = Integer.parseInt(decryptedTokenElements[0]); + this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); + this.queryStartTime = Long.parseLong(decryptedTokenElements[2]); + this.nameOfLastRespondedIndex = decryptedTokenElements[3]; if (posToStartPage < 0 || creationTimeOfLastRespondedIndex < 0 || queryStartTime < 0) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } @@ -182,11 +170,9 @@ public IndexStrategyPageToken( @Override public String generateEncryptedToken() { - return Base64.getEncoder() - .encodeToString( - (posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex) - .getBytes(UTF_8) - ); + return PageToken.encryptStringToken( + posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex + ); } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java index 0699ee5acea65..bc0e032263c35 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java @@ -8,6 +8,13 @@ package org.opensearch.rest.pagination; +import org.opensearch.OpenSearchParseException; + +import java.util.Base64; +import java.util.Objects; + +import static java.nio.charset.StandardCharsets.UTF_8; + /** * To be implemented by tokens getting returned/generated by {@link PaginationStrategy}. * @@ -15,5 +22,26 @@ */ public interface PageToken { + String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = + "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; + String generateEncryptedToken(); + + static String encryptStringToken(String tokenString) { + if (Objects.isNull(tokenString)) { + return null; + } + return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8)); + } + + static String decryptStringToken(String encTokenString) { + if (Objects.isNull(encTokenString)) { + return null; + } + try { + return new String(Base64.getDecoder().decode(encTokenString), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index c5168101e6a33..accc671d9cdb1 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -8,7 +8,10 @@ package org.opensearch.rest.pagination; +import org.opensearch.cluster.ClusterState; + import java.util.List; +import java.util.stream.Collectors; /** * Interface to be implemented by any strategy getting used for paginating rest responses. @@ -17,6 +20,8 @@ */ public interface PaginationStrategy { + String DESCENDING_SORT_PARAM_VALUE = "descending"; + /** * * @return Base64 encoded string, which can be used to fetch next page of response. @@ -28,4 +33,34 @@ public interface PaginationStrategy { * @return List of elements fetched corresponding to the store and token received by the strategy. */ List getElementsFromRequestedToken(); + + /** + * + * Utility method to get list of indices sorted by their creation time with {@param latestValidIndexCreateTime} + * being used to filter out the indices created after it. + */ + static List getListOfIndicesSortedByCreateTime( + final ClusterState clusterState, + String sortOrder, + final long latestValidIndexCreateTime + ) { + // Filter out the indices which have been created after the latest index which was present when + // paginated query started. Also, sort the indices list based on their creation timestamps + return clusterState.getRoutingTable() + .getIndicesRouting() + .keySet() + .stream() + .filter(index -> (latestValidIndexCreateTime >= clusterState.metadata().indices().get(index).getCreationDate())) + .sorted((index1, index2) -> { + Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); + Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); + if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { + return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) ? index2.compareTo(index1) : index1.compareTo(index2); + } + return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) + : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); + }) + .collect(Collectors.toList()); + } } diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java index 45653e9d8e4d6..18c9e987edc3f 100644 --- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java +++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java @@ -247,7 +247,7 @@ protected RestChannelConsumer doCatRequest(RestRequest request, NodeClient clien } @Override - protected void documentation(StringBuilder sb) { + public void documentation(StringBuilder sb) { } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 3e30d000f9c07..7055c51059e58 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -137,12 +137,14 @@ public void testBuildTable() { } } - final Table table = RestIndicesAction.RestIndicesActionCommonUtils.buildTable( + final RestIndicesAction action = new RestIndicesAction(); + final Table table = action.buildTable( new FakeRestRequest(), indicesSettings, indicesHealths, indicesStats, indicesMetadatas, + null, null ); diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java new file mode 100644 index 0000000000000..e931a89d0e813 --- /dev/null +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java @@ -0,0 +1,146 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; + +public class IndexBasedPaginationStrategyTests extends OpenSearchTestCase { + + public void testRetrieveAllIndicesInAscendingOrder() { + ClusterState clusterState = getRandomClusterState(); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(null, 1, "ascending", clusterState); + + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "ascending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "ascending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "ascending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNull(paginationStrategy.getNextToken()); + } + + public void testRetrieveAllIndicesInDescendingOrder() { + ClusterState clusterState = getRandomClusterState(); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(null, 1, "descending", clusterState); + + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "descending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "descending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getNextToken()); + + paginationStrategy = new IndexBasedPaginationStrategy( + (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), + 1, + "descending", + clusterState + ); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNull(paginationStrategy.getNextToken()); + } + + private ClusterState getRandomClusterState() { + final IndexMetadata indexMetadata1 = IndexMetadata.builder("test-index-1") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 1)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + final IndexRoutingTable.Builder indexRoutingTable1 = new IndexRoutingTable.Builder(indexMetadata1.getIndex()); + + final IndexMetadata indexMetadata2 = IndexMetadata.builder("test-index-2") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 2)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + final IndexRoutingTable.Builder indexRoutingTable2 = new IndexRoutingTable.Builder(indexMetadata2.getIndex()); + + final IndexMetadata indexMetadata3 = IndexMetadata.builder("test-index-3") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 3)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + final IndexRoutingTable.Builder indexRoutingTable3 = new IndexRoutingTable.Builder(indexMetadata3.getIndex()); + + final IndexMetadata indexMetadata4 = IndexMetadata.builder("test-index-4") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 4)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + final IndexRoutingTable.Builder indexRoutingTable4 = new IndexRoutingTable.Builder(indexMetadata4.getIndex()); + + Metadata metadata = Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata2, true) + .put(indexMetadata3, true) + .put(indexMetadata4, true) + .build(); + RoutingTable routingTable = RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable2) + .add(indexRoutingTable3) + .add(indexRoutingTable4) + .build(); + + return ClusterState.builder(new ClusterName("test")).metadata(metadata).routingTable(routingTable).build(); + } +} From f1b50166f5606b94da2caa712f014d0d931a0069 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Wed, 4 Sep 2024 05:57:32 +0530 Subject: [PATCH 04/12] Added paginated query request and response classes and removed page token interface Signed-off-by: Harsh Garg --- .../java/org/opensearch/common/Table.java | 56 +-- .../java/org/opensearch/rest/RestRequest.java | 12 + .../rest/action/cat/AbstractCatAction.java | 37 +- .../rest/action/cat/RestIndicesAction.java | 35 +- .../opensearch/rest/action/cat/RestTable.java | 18 +- .../action/list/RestIndicesListAction.java | 57 ++- .../IndexBasedPaginationStrategy.java | 130 +++--- .../opensearch/rest/pagination/PageToken.java | 47 --- .../pagination/PaginatedQueryRequest.java | 45 +++ .../pagination/PaginatedQueryResponse.java | 42 ++ .../rest/pagination/PaginationStrategy.java | 27 +- .../IndexBasedPaginationStrategyTests.java | 380 ++++++++++++++---- 12 files changed, 574 insertions(+), 312 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/rest/pagination/PageToken.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index a2d0e7990dac2..1ba238f8de45b 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -34,6 +34,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.core.common.Strings; +import org.opensearch.rest.pagination.PaginatedQueryResponse; import java.time.Instant; import java.time.ZoneOffset; @@ -43,8 +44,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import reactor.util.annotation.NonNull; - import static java.util.Collections.emptyMap; /** @@ -61,16 +60,17 @@ public class Table { private List currentCells; private boolean inHeaders = false; private boolean withTime = false; - private PaginationMetadata paginationMetadata = new PaginationMetadata(false, null, null); + /** + * paginatedQueryResponse if null will imply the Table response is not paginated. + */ + private PaginatedQueryResponse paginatedQueryResponse; public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; public Table() {} - public Table(@Nullable PaginationMetadata paginationMetadata) { - if (paginationMetadata != null) { - this.paginationMetadata = paginationMetadata; - } + public Table(@Nullable PaginatedQueryResponse paginatedQueryResponse) { + this.paginatedQueryResponse = paginatedQueryResponse; } public Table startHeaders() { @@ -241,16 +241,8 @@ public Map getAliasMap() { return headerAliasMap; } - public boolean isPaginated() { - return paginationMetadata.isResponsePaginated; - } - - public String getPaginatedElement() { - return paginationMetadata.paginatedElement; - } - - public String getNextToken() { - return paginationMetadata.nextToken; + public PaginatedQueryResponse getPaginatedQueryResponse() { + return paginatedQueryResponse; } /** @@ -277,34 +269,4 @@ public Cell(Object value, Map attr) { this.attr = attr; } } - - /** - * Pagination metadata for a table. - * - * @opensearch.internal - */ - public static class PaginationMetadata { - - /** - * boolean denoting whether the table is paginated or not. - */ - public final boolean isResponsePaginated; - - /** - * String denoting the element which is being paginated (for e.g. shards, indices..). - */ - public final String paginatedElement; - - /** - * String denoting the next_token of paginated response, which will be used to fetch next page (if any). - */ - public final String nextToken; - - public PaginationMetadata(@NonNull boolean isResponsePaginated, @Nullable String paginatedElement, @Nullable String nextToken) { - this.isResponsePaginated = isResponsePaginated; - assert !isResponsePaginated || paginatedElement != null : "paginatedElement must be specified for a table which is paginated"; - this.paginatedElement = paginatedElement; - this.nextToken = nextToken; - } - } } diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 2c397f7fc6e8e..1b9f5708a00fe 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -51,6 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; +import org.opensearch.rest.pagination.PaginatedQueryRequest; import java.io.IOException; import java.io.InputStream; @@ -67,6 +68,9 @@ import static org.opensearch.common.unit.TimeValue.parseTimeValue; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY; +import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SIZE_KEY; +import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SORT_KEY; /** * REST Request @@ -591,6 +595,14 @@ public static MediaType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } + public PaginatedQueryRequest parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) { + return new PaginatedQueryRequest( + param(PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY), + param(PAGINATED_QUERY_PARAM_SORT_KEY, defaultSortOrder), + paramAsInt(PAGINATED_QUERY_PARAM_SIZE_KEY, defaultPageSize) + ); + } + /** * Thrown if there is an error in the content type header. * diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index 6a93d10a7a8e4..c85a458d0f0c6 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -40,13 +40,12 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.pagination.PageToken; +import org.opensearch.rest.pagination.PaginatedQueryRequest; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.Objects; import java.util.Set; @@ -60,7 +59,7 @@ */ public abstract class AbstractCatAction extends BaseRestHandler { - protected PaginationQueryMetadata paginationQueryMetadata; + protected PaginatedQueryRequest paginatedQueryRequest; protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client); @@ -91,8 +90,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC }; } else { if (isActionPaginated()) { - this.paginationQueryMetadata = validateAndGetPaginationMetadata(request); - assert Objects.nonNull(paginationQueryMetadata) : "paginationQueryMetadata can not be null for paginated queries"; + this.paginatedQueryRequest = validateAndGetPaginationMetadata(request); + assert Objects.nonNull(paginatedQueryRequest) : "paginatedQueryRequest can not be null for paginated queries"; } return doCatRequest(request, client); } @@ -120,34 +119,10 @@ public boolean isActionPaginated() { * * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide * its own implementation. Query params supported by the action specific to pagination along with the respective validations, - * should be added here. The actions would also use the {@param restRequest} to initialise a {@link PageToken}. + * should be added here. */ - protected PaginationQueryMetadata validateAndGetPaginationMetadata(RestRequest restRequest) { + protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest restRequest) { return null; } - /** - * A pagination helper class which would contain requested page token and - * a map of query params required by a paginated API. - * - * @opensearch.internal - */ - public static class PaginationQueryMetadata { - private final Map paginationQueryParams; - private final PageToken requestedPageToken; - - public PaginationQueryMetadata(final Map paginationQueryParams, PageToken requestedPageToken) { - this.paginationQueryParams = paginationQueryParams; - this.requestedPageToken = requestedPageToken; - } - - public Map getPaginationQueryParams() { - return paginationQueryParams; - } - - public PageToken getRequestedPageToken() { - return requestedPageToken; - } - } - } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index e2212ca2cdd30..594444a253821 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -62,6 +62,7 @@ import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; +import org.opensearch.rest.pagination.PaginatedQueryResponse; import java.time.Instant; import java.time.ZoneOffset; @@ -152,17 +153,14 @@ public RestResponse buildResponse(final Table table) throws Exception { @Override public void onResponse(final ClusterStateResponse clusterStateResponse) { IndexBasedPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); - Table.PaginationMetadata paginationMetadata = getTablePaginationMetadata(paginationStrategy); - final String[] indicesToBeQueried = isActionPaginated() - ? paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]) - : indices; + final String[] indicesToBeQueried = getIndicesToBeQueried(indices, paginationStrategy); final GroupedActionListener groupedListener = createGroupedListener( request, 4, listener, indicesToBeQueried, - paginationMetadata + getPaginatedQueryResponse(paginationStrategy) ); groupedListener.onResponse(clusterStateResponse); @@ -172,20 +170,19 @@ public void onResponse(final ClusterStateResponse clusterStateResponse) { // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the get settings request might be deleted when the subsequent - // cluster - // state, cluster health and indices stats requests execute. We have to distinguish two cases: + // Indices that were successfully resolved during the cluster state request might be deleted when the subsequent + // get settings, cluster health and indices stats requests execute. We have to distinguish two cases: // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the // subsequent requests to fail. // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests // not to // fail on the deleted index (as we want to ignore wildcards that cannot be resolved). - // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests re-resolve + // This behavior can be ensured by letting the get settings, cluster health and indices stats requests re-resolve // the // index names with the same indices options that we used for the initial cluster state request (strictExpand). sendGetSettingsRequest( indicesToBeQueried, - subRequestIndicesOptions, + indicesOptions, local, clusterManagerNodeTimeout, client, @@ -301,7 +298,7 @@ private GroupedActionListener createGroupedListener( final int size, final ActionListener
listener, final String[] indicesToBeQueried, - final Table.PaginationMetadata paginationMetadata + final PaginatedQueryResponse paginatedQueryResponse ) { return new GroupedActionListener<>(new ActionListener>() { @Override @@ -332,7 +329,7 @@ public void onResponse(final Collection responses) { indicesStats, indicesStates, indicesToBeQueried, - paginationMetadata + paginatedQueryResponse ); listener.onResponse(responseTable); } catch (Exception e) { @@ -365,8 +362,8 @@ protected Table getTableWithHeader(final RestRequest request) { return getTableWithHeader(request, null); } - protected Table getTableWithHeader(final RestRequest request, final Table.PaginationMetadata paginationMetadata) { - Table table = new Table(paginationMetadata); + protected Table getTableWithHeader(final RestRequest request, final PaginatedQueryResponse paginatedQueryResponse) { + Table table = new Table(paginatedQueryResponse); table.startHeaders(); table.addCell("health", "alias:h;desc:current health status"); table.addCell("status", "alias:s;desc:open/close status"); @@ -737,11 +734,11 @@ Table buildTable( final Map indicesStats, final Map indicesMetadatas, final String[] indicesToBeQueried, - final Table.PaginationMetadata paginationMetadata + final PaginatedQueryResponse paginatedQueryResponse ) { final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request, paginationMetadata); + final Table table = getTableWithHeader(request, paginatedQueryResponse); if (isActionPaginated() && indicesToBeQueried.length == 0) { // to handle cases where paginationStrategy couldn't find any indices that should be queried @@ -1030,8 +1027,12 @@ protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateRespons return null; } - protected Table.PaginationMetadata getTablePaginationMetadata(IndexBasedPaginationStrategy paginationStrategy) { + protected PaginatedQueryResponse getPaginatedQueryResponse(IndexBasedPaginationStrategy paginationStrategy) { return null; } + protected String[] getIndicesToBeQueried(String[] indices, IndexBasedPaginationStrategy paginationStrategy) { + return indices; + } + } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 1eecc70b7e213..55ad21a4fc19c 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -58,8 +58,11 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; +import static org.opensearch.rest.pagination.PaginatedQueryResponse.PAGINATED_RESPONSE_NEXT_TOKEN_KEY; + /** * a REST table * @@ -88,11 +91,12 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel XContentBuilder builder = channel.newBuilder(); List displayHeaders = buildDisplayHeaders(table, request); - if (table.isPaginated()) { - assert table.getPaginatedElement() != null : "Paginated element is required in-case nextToken is not null"; + if (Objects.nonNull(table.getPaginatedQueryResponse())) { + assert Objects.nonNull(table.getPaginatedQueryResponse().getPaginatedElement()) + : "Paginated element is required in-case of paginated responses"; builder.startObject(); - builder.field("next_token", table.getNextToken()); - builder.startArray(table.getPaginatedElement()); + builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPaginatedQueryResponse().getNextToken()); + builder.startArray(table.getPaginatedQueryResponse().getPaginatedElement()); } else { builder.startArray(); } @@ -105,7 +109,7 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel builder.endObject(); } builder.endArray(); - if (table.isPaginated()) { + if (Objects.nonNull(table.getPaginatedQueryResponse())) { builder.endObject(); } return new BytesRestResponse(RestStatus.OK, builder); @@ -147,8 +151,8 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann out.append("\n"); } // Adding a new row for next_token, in the response if the table is paginated. - if (table.isPaginated()) { - out.append("next_token" + " " + table.getNextToken()); + if (Objects.nonNull(table.getPaginatedQueryResponse())) { + out.append("next_token" + " " + table.getPaginatedQueryResponse().getNextToken()); out.append("\n"); } out.close(); diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index b120f308f3d5e..07e6b87ac9aa2 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,14 +9,13 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.common.Table; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; +import org.opensearch.rest.pagination.PaginatedQueryRequest; +import org.opensearch.rest.pagination.PaginatedQueryResponse; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import static java.util.Arrays.asList; @@ -56,49 +55,41 @@ public boolean isActionPaginated() { } @Override - protected PaginationQueryMetadata validateAndGetPaginationMetadata(RestRequest restRequest) { - Map paginatedQueryParams = new HashMap<>(); - final String requestedTokenStr = restRequest.param("next_token"); - final String sortOrder = restRequest.param("sort", "ascending"); - final int pageSize = restRequest.paramAsInt("size", DEFAULT_LIST_INDICES_PAGE_SIZE_STRING); + protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest restRequest) { + PaginatedQueryRequest paginatedQueryRequest = restRequest.parsePaginatedQueryParams( + "ascending", + DEFAULT_LIST_INDICES_PAGE_SIZE_STRING + ); // validating pageSize - if (pageSize <= 0) { + if (paginatedQueryRequest.getSize() <= 0) { throw new IllegalArgumentException("size must be greater than zero"); - } else if (pageSize > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { + } else if (paginatedQueryRequest.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING + "]"); } // Validating sort order - if (!Objects.equals(sortOrder, "ascending") && !Objects.equals(sortOrder, "descending")) { + if (!Objects.equals(paginatedQueryRequest.getSort(), "ascending") + && !Objects.equals(paginatedQueryRequest.getSort(), "descending")) { throw new IllegalArgumentException("value of sort can either be ascending or descending"); } - // Next Token in the request will be validated by the IndexStrategyPageToken itself. - IndexBasedPaginationStrategy.IndexStrategyPageToken requestedPageToken = requestedTokenStr == null - ? null - : new IndexBasedPaginationStrategy.IndexStrategyPageToken(requestedTokenStr); - paginatedQueryParams.put("next_token", requestedTokenStr); - paginatedQueryParams.put("size", pageSize); - paginatedQueryParams.put("sort", sortOrder); - return new PaginationQueryMetadata(paginatedQueryParams, requestedPageToken); + // Next Token in the request will be validated by the IndexStrategyTokenParser itself. + if (Objects.nonNull(paginatedQueryRequest.getRequestedTokenStr())) { + IndexBasedPaginationStrategy.IndexStrategyTokenParser.validateIndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); + } + + return paginatedQueryRequest; } @Override protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { - assert !Objects.nonNull(paginationQueryMetadata.getRequestedPageToken()) - || paginationQueryMetadata.getRequestedPageToken() instanceof IndexBasedPaginationStrategy.IndexStrategyPageToken; - return new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationQueryMetadata.getRequestedPageToken(), - (int) paginationQueryMetadata.getPaginationQueryParams().get("size"), - (String) paginationQueryMetadata.getPaginationQueryParams().get("sort"), - clusterStateResponse.getState() - ); + return new IndexBasedPaginationStrategy(paginatedQueryRequest, PAGINATED_LIST_INDICES_ELEMENT_KEY, clusterStateResponse.getState()); } @Override - protected Table.PaginationMetadata getTablePaginationMetadata(IndexBasedPaginationStrategy paginationStrategy) { - return new Table.PaginationMetadata( - true, - PAGINATED_LIST_INDICES_ELEMENT_KEY, - paginationStrategy.getNextToken() == null ? null : paginationStrategy.getNextToken().generateEncryptedToken() - ); + protected PaginatedQueryResponse getPaginatedQueryResponse(IndexBasedPaginationStrategy paginationStrategy) { + return paginationStrategy.getPaginatedQueryResponse(); + } + + protected String[] getIndicesToBeQueried(String[] indices, IndexBasedPaginationStrategy paginationStrategy) { + return paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]); } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java index f79c56e88d994..8fe06c8fee891 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -24,58 +24,64 @@ */ public class IndexBasedPaginationStrategy implements PaginationStrategy { - private final IndexStrategyPageToken nextToken; + private final PaginatedQueryResponse paginatedQueryResponse; private final List indicesFromRequestedToken; - public IndexBasedPaginationStrategy( - @Nullable IndexStrategyPageToken requestedToken, - int maxPageSize, - String sortOrder, - ClusterState clusterState - ) { + public IndexBasedPaginationStrategy(PaginatedQueryRequest paginatedQueryRequest, String paginatedElement, ClusterState clusterState) { + IndexStrategyTokenParser requestedToken = paginatedQueryRequest.getRequestedTokenStr() == null + ? null + : new IndexStrategyTokenParser(paginatedQueryRequest.getRequestedTokenStr()); + // Get sorted list of indices not containing the ones created after query start time List sortedIndicesList = PaginationStrategy.getListOfIndicesSortedByCreateTime( clusterState, - sortOrder, + paginatedQueryRequest.getSort(), requestedToken == null ? Long.MAX_VALUE : requestedToken.queryStartTime ); if (sortedIndicesList.isEmpty()) { // Denotes, that all the indices which were created before the queryStartTime have been deleted. // No nextToken and indices need to be shown in such cases. this.indicesFromRequestedToken = new ArrayList<>(); - this.nextToken = null; + this.paginatedQueryResponse = new PaginatedQueryResponse(null, paginatedElement); } else { final int requestedPageStartIndexNumber = getRequestedPageIndexStartNumber( requestedToken, sortedIndicesList, clusterState, - sortOrder + paginatedQueryRequest.getSort() ); // inclusive - int requestedPageEndIndexNumber = Math.min(requestedPageStartIndexNumber + maxPageSize, sortedIndicesList.size()); // exclusive + int requestedPageEndIndexNumber = Math.min( + requestedPageStartIndexNumber + paginatedQueryRequest.getSize(), + sortedIndicesList.size() + ); // exclusive this.indicesFromRequestedToken = sortedIndicesList.subList(requestedPageStartIndexNumber, requestedPageEndIndexNumber); // Set the queryStart time as the timestamp of latest created index if requested token is null. long queryStartTime = requestedToken == null - ? DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + ? DESCENDING_SORT_PARAM_VALUE.equals(paginatedQueryRequest.getSort()) ? clusterState.metadata().indices().get(sortedIndicesList.get(0)).getCreationDate() : clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() : requestedToken.queryStartTime; - this.nextToken = requestedPageEndIndexNumber >= sortedIndicesList.size() - ? null - : new IndexStrategyPageToken( - requestedPageEndIndexNumber, - clusterState.metadata().indices().get(sortedIndicesList.get(requestedPageEndIndexNumber - 1)).getCreationDate(), - queryStartTime, - sortedIndicesList.get(requestedPageEndIndexNumber - 1) - ); + + this.paginatedQueryResponse = new PaginatedQueryResponse( + requestedPageEndIndexNumber >= sortedIndicesList.size() + ? null + : new IndexStrategyTokenParser( + requestedPageEndIndexNumber, + clusterState.metadata().indices().get(sortedIndicesList.get(requestedPageEndIndexNumber - 1)).getCreationDate(), + queryStartTime, + sortedIndicesList.get(requestedPageEndIndexNumber - 1) + ).generateEncryptedToken(), + paginatedElement + ); } } @Override @Nullable - public PageToken getNextToken() { - return nextToken; + public PaginatedQueryResponse getPaginatedQueryResponse() { + return paginatedQueryResponse; } @Override @@ -84,17 +90,17 @@ public List getElementsFromRequestedToken() { } private int getRequestedPageIndexStartNumber( - final IndexStrategyPageToken requestedPageToken, + final IndexStrategyTokenParser requestedTokenParser, final List sortedIndicesList, final ClusterState clusterState, final String sortOrder ) { - if (Objects.isNull(requestedPageToken)) { + if (Objects.isNull(requestedTokenParser)) { return 0; } - int requestedPageStartIndexNumber = Math.min(requestedPageToken.posToStartPage, sortedIndicesList.size() - 1); + int requestedPageStartIndexNumber = Math.min(requestedTokenParser.posToStartPage, sortedIndicesList.size()); if (requestedPageStartIndexNumber > 0 - && !Objects.equals(sortedIndicesList.get(requestedPageStartIndexNumber - 1), requestedPageToken.nameOfLastRespondedIndex)) { + && !Objects.equals(sortedIndicesList.get(requestedPageStartIndexNumber - 1), requestedTokenParser.nameOfLastRespondedIndex)) { // case denoting an already responded index has been deleted while the paginated queries are being executed // find the index whose creation time is just after/before (based on sortOrder) the index which was last responded. while (requestedPageStartIndexNumber > 0) { @@ -102,10 +108,10 @@ private int getRequestedPageIndexStartNumber( .indices() .get(sortedIndicesList.get(requestedPageStartIndexNumber - 1)) .getCreationDate(); - if ((DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - && creationTimeOfIndex < requestedPageToken.creationTimeOfLastRespondedIndex) + if ((!DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + && creationTimeOfIndex < requestedTokenParser.creationTimeOfLastRespondedIndex) || (DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - && creationTimeOfIndex > requestedPageToken.creationTimeOfLastRespondedIndex)) { + && creationTimeOfIndex > requestedTokenParser.creationTimeOfLastRespondedIndex)) { break; } requestedPageStartIndexNumber--; @@ -115,17 +121,46 @@ private int getRequestedPageIndexStartNumber( } /** - * Token to be used by {@link IndexBasedPaginationStrategy}. + * TokenParser to be used by {@link IndexBasedPaginationStrategy}. * Token would like: IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + * QueryStartTime + $ + NameOfLastRespondedIndex */ - public static class IndexStrategyPageToken implements PageToken { + public static class IndexStrategyTokenParser { private final int posToStartPage; private final long creationTimeOfLastRespondedIndex; private final long queryStartTime; private final String nameOfLastRespondedIndex; + public IndexStrategyTokenParser(String requestedTokenString) { + validateIndexStrategyToken(requestedTokenString); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); + final String[] decryptedTokenElements = decryptedToken.split("\\$"); + this.posToStartPage = Integer.parseInt(decryptedTokenElements[0]); + this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); + this.queryStartTime = Long.parseLong(decryptedTokenElements[2]); + this.nameOfLastRespondedIndex = decryptedTokenElements[3]; + } + + public IndexStrategyTokenParser( + int indexNumberToStartPageFrom, + long creationTimeOfLastRespondedIndex, + long queryStartTime, + String nameOfLastRespondedIndex + ) { + Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); + this.posToStartPage = indexNumberToStartPageFrom; + this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; + this.queryStartTime = queryStartTime; + this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; + } + + public String generateEncryptedToken() { + return PaginationStrategy.encryptStringToken( + posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex + ); + } + /** * Will perform simple validations on token received in the request and initialize the data members. * The token should be base64 encoded, and should contain the expected number of elements separated by "$". @@ -133,20 +168,17 @@ public static class IndexStrategyPageToken implements PageToken { * * @param requestedTokenString string denoting the encoded next token requested by the user */ - public IndexStrategyPageToken(String requestedTokenString) { + public static void validateIndexStrategyToken(String requestedTokenString) { Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); - String decryptedToken = PageToken.decryptStringToken(requestedTokenString); - + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); final String[] decryptedTokenElements = decryptedToken.split("\\$"); if (decryptedTokenElements.length != 4) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } - try { - this.posToStartPage = Integer.parseInt(decryptedTokenElements[0]); - this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); - this.queryStartTime = Long.parseLong(decryptedTokenElements[2]); - this.nameOfLastRespondedIndex = decryptedTokenElements[3]; + int posToStartPage = Integer.parseInt(decryptedTokenElements[0]); + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); + long queryStartTime = Long.parseLong(decryptedTokenElements[2]); if (posToStartPage < 0 || creationTimeOfLastRespondedIndex < 0 || queryStartTime < 0) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } @@ -154,26 +186,6 @@ public IndexStrategyPageToken(String requestedTokenString) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } } - - public IndexStrategyPageToken( - int indexNumberToStartPageFrom, - long creationTimeOfLastRespondedIndex, - long queryStartTime, - String nameOfLastRespondedIndex - ) { - Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); - this.posToStartPage = indexNumberToStartPageFrom; - this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; - this.queryStartTime = queryStartTime; - this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; - } - - @Override - public String generateEncryptedToken() { - return PageToken.encryptStringToken( - posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex - ); - } } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java deleted file mode 100644 index bc0e032263c35..0000000000000 --- a/server/src/main/java/org/opensearch/rest/pagination/PageToken.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.pagination; - -import org.opensearch.OpenSearchParseException; - -import java.util.Base64; -import java.util.Objects; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * To be implemented by tokens getting returned/generated by {@link PaginationStrategy}. - * - * @opensearch.internal - */ -public interface PageToken { - - String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = - "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; - - String generateEncryptedToken(); - - static String encryptStringToken(String tokenString) { - if (Objects.isNull(tokenString)) { - return null; - } - return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8)); - } - - static String decryptStringToken(String encTokenString) { - if (Objects.isNull(encTokenString)) { - return null; - } - try { - return new String(Base64.getDecoder().decode(encTokenString), UTF_8); - } catch (IllegalArgumentException exception) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } - } -} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java new file mode 100644 index 0000000000000..cfcf93d6ce95c --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.common.annotation.PublicApi; + +/** + * + * Class specific to paginated queries, which will contain common query params required by a paginated API. + */ +@PublicApi(since = "2.17.0") +public class PaginatedQueryRequest { + + public static final String PAGINATED_QUERY_PARAM_SORT_KEY = "sort"; + public static final String PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY = "next_token"; + public static final String PAGINATED_QUERY_PARAM_SIZE_KEY = "size"; + private final String requestedTokenStr; + private final String sort; + private final int size; + + public PaginatedQueryRequest(String requested_token, String sort, int size) { + this.requestedTokenStr = requested_token; + this.sort = sort; + this.size = size; + } + + public String getSort() { + return sort; + } + + public String getRequestedTokenStr() { + return requestedTokenStr; + } + + public int getSize() { + return size; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java new file mode 100644 index 0000000000000..b9abe839b7230 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +/** + * Pagination response metadata for a paginated query. + * @opensearch.internal + */ +public class PaginatedQueryResponse { + + public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; + + /** + * String denoting the next_token of paginated response, which will be used to fetch next page (if any). + */ + private final String nextToken; + + /** + * String denoting the element which is being paginated (for e.g. shards, indices..). + */ + private final String paginatedElement; + + public PaginatedQueryResponse(String nextToken, String paginatedElement) { + assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; + this.nextToken = nextToken; + this.paginatedElement = paginatedElement; + } + + public String getNextToken() { + return nextToken; + } + + public String getPaginatedElement() { + return paginatedElement; + } +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index accc671d9cdb1..e9d25d38eeaa9 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -8,11 +8,16 @@ package org.opensearch.rest.pagination; +import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; +import java.util.Base64; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Interface to be implemented by any strategy getting used for paginating rest responses. * @@ -21,12 +26,14 @@ public interface PaginationStrategy { String DESCENDING_SORT_PARAM_VALUE = "descending"; + String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = + "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; /** * * @return Base64 encoded string, which can be used to fetch next page of response. */ - PageToken getNextToken(); + PaginatedQueryResponse getPaginatedQueryResponse(); /** * @@ -63,4 +70,22 @@ static List getListOfIndicesSortedByCreateTime( }) .collect(Collectors.toList()); } + + static String encryptStringToken(String tokenString) { + if (Objects.isNull(tokenString)) { + return null; + } + return Base64.getEncoder().encodeToString(tokenString.getBytes(UTF_8)); + } + + static String decryptStringToken(String encTokenString) { + if (Objects.isNull(encTokenString)) { + return null; + } + try { + return new String(Base64.getDecoder().decode(encTokenString), UTF_8); + } catch (IllegalArgumentException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java index e931a89d0e813..b85a1a44d5074 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java @@ -8,6 +8,7 @@ package org.opensearch.rest.pagination; +import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -21,113 +22,353 @@ public class IndexBasedPaginationStrategyTests extends OpenSearchTestCase { + private final IndexMetadata indexMetadata1 = IndexMetadata.builder("test-index-1") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 1)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + private final IndexRoutingTable.Builder indexRoutingTable1 = new IndexRoutingTable.Builder(indexMetadata1.getIndex()); + private final IndexMetadata indexMetadata2 = IndexMetadata.builder("test-index-2") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 2)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + private final IndexRoutingTable.Builder indexRoutingTable2 = new IndexRoutingTable.Builder(indexMetadata2.getIndex()); + private final IndexMetadata indexMetadata3 = IndexMetadata.builder("test-index-3") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 3)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + private final IndexRoutingTable.Builder indexRoutingTable3 = new IndexRoutingTable.Builder(indexMetadata3.getIndex()); + private final IndexMetadata indexMetadata4 = IndexMetadata.builder("test-index-4") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 4)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + private final IndexRoutingTable.Builder indexRoutingTable4 = new IndexRoutingTable.Builder(indexMetadata4.getIndex()); + private final IndexMetadata indexMetadata5 = IndexMetadata.builder("test-index-5") + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 5)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + private final IndexRoutingTable.Builder indexRoutingTable5 = new IndexRoutingTable.Builder(indexMetadata5.getIndex()); + public void testRetrieveAllIndicesInAscendingOrder() { ClusterState clusterState = getRandomClusterState(); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(null, 1, "ascending", clusterState); - + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "ascending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "ascending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "ascending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getNextToken()); + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); } public void testRetrieveAllIndicesInDescendingOrder() { ClusterState clusterState = getRandomClusterState(); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(null, 1, "descending", clusterState); - + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "descending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "descending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getNextToken()); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - paginationStrategy = new IndexBasedPaginationStrategy( - (IndexBasedPaginationStrategy.IndexStrategyPageToken) paginationStrategy.getNextToken(), - 1, - "descending", - clusterState - ); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getNextToken()); + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); } - private ClusterState getRandomClusterState() { - final IndexMetadata indexMetadata1 = IndexMetadata.builder("test-index-1") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 1)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) + public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4) + ClusterState clusterState = getRandomClusterState(); + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Query2 adding index5 to clusterState + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata2, true) + .put(indexMetadata3, true) + .put(indexMetadata4, true) + .put(indexMetadata5, true) + .build() + ) + .routingTable( + RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable2) + .add(indexRoutingTable3) + .add(indexRoutingTable4) + .add(indexRoutingTable5) + .build() + ) .build(); - final IndexRoutingTable.Builder indexRoutingTable1 = new IndexRoutingTable.Builder(indexMetadata1.getIndex()); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - final IndexMetadata indexMetadata2 = IndexMetadata.builder("test-index-2") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 2)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) + // Deleting index2 which has already been displayed, still index3 should get displayed + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata3, true) + .put(indexMetadata4, true) + .put(indexMetadata5, true) + .build() + ) + .routingTable( + RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable3) + .add(indexRoutingTable4) + .add(indexRoutingTable5) + .build() + ) .build(); - final IndexRoutingTable.Builder indexRoutingTable2 = new IndexRoutingTable.Builder(indexMetadata2.getIndex()); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - final IndexMetadata indexMetadata3 = IndexMetadata.builder("test-index-3") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 3)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) + // Deleting index4 which is not yet displayed which otherwise should have been displayed in the following query + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(indexMetadata1, true).put(indexMetadata3, true).put(indexMetadata5, true).build()) + .routingTable(RoutingTable.builder().add(indexRoutingTable1).add(indexRoutingTable3).add(indexRoutingTable5).build()) .build(); - final IndexRoutingTable.Builder indexRoutingTable3 = new IndexRoutingTable.Builder(indexMetadata3.getIndex()); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } - final IndexMetadata indexMetadata4 = IndexMetadata.builder("test-index-4") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 4)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) + public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDescOrder() { + // Query1 with 4 indices in clusterState (test-index1,2,3,4) + ClusterState clusterState = getRandomClusterState(); + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Query2 adding index5 to clusterState + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata2, true) + .put(indexMetadata3, true) + .put(indexMetadata4, true) + .put(indexMetadata5, true) + .build() + ) + .routingTable( + RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable2) + .add(indexRoutingTable3) + .add(indexRoutingTable4) + .add(indexRoutingTable5) + .build() + ) .build(); - final IndexRoutingTable.Builder indexRoutingTable4 = new IndexRoutingTable.Builder(indexMetadata4.getIndex()); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + // Deleting index3 which has already been displayed, still index2 should get displayed + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata2, true) + .put(indexMetadata4, true) + .put(indexMetadata5, true) + .build() + ) + .routingTable( + RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable2) + .add(indexRoutingTable4) + .add(indexRoutingTable5) + .build() + ) + .build(); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Deleting index1 which is not yet displayed which otherwise should have been displayed in the following query + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(indexMetadata2, true).put(indexMetadata4, true).put(indexMetadata5, true).build()) + .routingTable(RoutingTable.builder().add(indexRoutingTable2).add(indexRoutingTable4).add(indexRoutingTable5).build()) + .build(); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } + + public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() { + // Query1 with 5 indices in clusterState (test-index1,2,3,4,5) + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .put(indexMetadata1, true) + .put(indexMetadata2, true) + .put(indexMetadata3, true) + .put(indexMetadata4, true) + .put(indexMetadata5, true) + .build() + ) + .routingTable( + RoutingTable.builder() + .add(indexRoutingTable1) + .add(indexRoutingTable2) + .add(indexRoutingTable3) + .add(indexRoutingTable4) + .add(indexRoutingTable5) + .build() + ) + .build(); + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Query2, no changes to clusterState + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Deleting index1,index2 & index3. index4 should get displayed + clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(indexMetadata4, true).put(indexMetadata5, true).build()) + .routingTable(RoutingTable.builder().add(indexRoutingTable4).add(indexRoutingTable5).build()) + .build(); + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + + // Deleting index1 which is not yet displayed which otherwise should have been displayed in the following query + paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } + + public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { + try { + new IndexBasedPaginationStrategy.IndexStrategyTokenParser(null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("requestedTokenString can not be null"); + } + } + + public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { + try { + // % is not allowed in base64 encoding + new IndexBasedPaginationStrategy.IndexStrategyTokenParser("3%4%5"); + fail("expected exception"); + } catch (OpenSearchParseException e) { + assert e.getMessage().contains("[next_token] has been tainted"); + } + } + + public void testIndexStrategyPageTokenWithLessElementsInRequestedToken() { + String encryptedRequestToken = PaginationStrategy.encryptStringToken("1$1725361543$1725361543"); + try { + // should throw exception as it expects 4 elements separated by $ + new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); + fail("expected exception"); + } catch (OpenSearchParseException e) { + assert e.getMessage().contains("[next_token] has been tainted"); + } + } + + public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { + { + String encryptedRequestToken = PaginationStrategy.encryptStringToken("1$-1725361543$-1725361543$index"); + try { + // should not accept invalid long values (negative) + new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); + fail("expected exception"); + } catch (OpenSearchParseException e) { + assert e.getMessage().contains("[next_token] has been tainted"); + } + } + + { + String encryptedRequestToken = PaginationStrategy.encryptStringToken("-1$1725361543$1725361543$index"); + try { + // should not accept invalid int values (negative) + new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); + fail("expected exception"); + } catch (OpenSearchParseException e) { + assert e.getMessage().contains("[next_token] has been tainted"); + } + } + } + + public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { + try { + new IndexBasedPaginationStrategy.IndexStrategyTokenParser(1, 1234l, 1234l, null); + fail("expected exception"); + } catch (Exception e) { + assert e.getMessage().contains("index name should be provided"); + } + } + + private ClusterState getRandomClusterState() { Metadata metadata = Metadata.builder() .put(indexMetadata1, true) .put(indexMetadata2, true) @@ -140,7 +381,6 @@ private ClusterState getRandomClusterState() { .add(indexRoutingTable3) .add(indexRoutingTable4) .build(); - return ClusterState.builder(new ClusterName("test")).metadata(metadata).routingTable(routingTable).build(); } } From 062197588878c96d77e6013f5bd78f4acb7583e3 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Wed, 11 Sep 2024 10:25:02 +0530 Subject: [PATCH 05/12] Removing queryStartTime from IndexStrategyToken Signed-off-by: Harsh Garg --- .../rest/action/cat/RestIndicesAction.java | 488 +++++++++--------- .../action/list/RestIndicesListAction.java | 29 +- .../IndexBasedPaginationStrategy.java | 222 ++++---- .../rest/pagination/PaginationStrategy.java | 36 +- .../IndexBasedPaginationStrategyTests.java | 290 +++-------- 5 files changed, 503 insertions(+), 562 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 594444a253821..6de67bb7d1add 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -143,65 +143,75 @@ public RestResponse buildResponse(final Table table) throws Exception { return RestTable.buildResponse(table, channel); } }); - sendClusterStateRequest( + + sendGetSettingsRequest( indices, indicesOptions, local, clusterManagerNodeTimeout, client, - new ActionListener() { + new ActionListener() { @Override - public void onResponse(final ClusterStateResponse clusterStateResponse) { - IndexBasedPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); - final String[] indicesToBeQueried = getIndicesToBeQueried(indices, paginationStrategy); - - final GroupedActionListener groupedListener = createGroupedListener( - request, - 4, - listener, - indicesToBeQueried, - getPaginatedQueryResponse(paginationStrategy) - ); - groupedListener.onResponse(clusterStateResponse); - + public void onResponse(final GetSettingsResponse getSettingsResponse) { // The list of indices that will be returned is determined by the indices returned from the Get Settings call. // All the other requests just provide additional detail, and wildcards may be resolved differently depending on the // type of request in the presence of security plugins (looking at you, ClusterHealthRequest), so // force the IndicesOptions for all the sub-requests to be as inclusive as possible. final IndicesOptions subRequestIndicesOptions = IndicesOptions.lenientExpandHidden(); - // Indices that were successfully resolved during the cluster state request might be deleted when the subsequent - // get settings, cluster health and indices stats requests execute. We have to distinguish two cases: - // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the - // subsequent requests to fail. - // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests - // not to - // fail on the deleted index (as we want to ignore wildcards that cannot be resolved). - // This behavior can be ensured by letting the get settings, cluster health and indices stats requests re-resolve - // the - // index names with the same indices options that we used for the initial cluster state request (strictExpand). - sendGetSettingsRequest( - indicesToBeQueried, - indicesOptions, - local, - clusterManagerNodeTimeout, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendIndicesStatsRequest( - indicesToBeQueried, - subRequestIndicesOptions, - includeUnloadedSegments, - client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) - ); - sendClusterHealthRequest( - indicesToBeQueried, + // Indices that were successfully resolved during the get settings request might be deleted when the + // subsequent cluster state, cluster health and indices stats requests execute. We have to distinguish two cases: + // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we + // want the subsequent requests to fail. + // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent + // requests not to fail on the deleted index (as we want to ignore wildcards that cannot be resolved). + // This behavior can be ensured by letting the cluster state, cluster health and indices stats requests + // re-resolve the index names with the same indices options that we used for the initial cluster state + // request (strictExpand). + sendClusterStateRequest( + indices, subRequestIndicesOptions, local, clusterManagerNodeTimeout, client, - ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + new ActionListener() { + @Override + public void onResponse(ClusterStateResponse clusterStateResponse) { + IndexBasedPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); + final String[] indicesToBeQueried = getIndicesToBeQueried(indices, paginationStrategy); + final GroupedActionListener groupedListener = createGroupedListener( + request, + 4, + listener, + indicesToBeQueried, + getPaginatedQueryResponse(paginationStrategy) + ); + groupedListener.onResponse(getSettingsResponse); + groupedListener.onResponse(clusterStateResponse); + + sendIndicesStatsRequest( + indicesToBeQueried, + subRequestIndicesOptions, + includeUnloadedSegments, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + + sendClusterHealthRequest( + indicesToBeQueried, + subRequestIndicesOptions, + local, + clusterManagerNodeTimeout, + client, + ActionListener.wrap(groupedListener::onResponse, groupedListener::onFailure) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + } ); } @@ -212,6 +222,7 @@ public void onFailure(final Exception e) { } ); }; + } /** @@ -727,7 +738,7 @@ protected Table getTableWithHeader(final RestRequest request, final PaginatedQue } // package private for testing - Table buildTable( + protected Table buildTable( final RestRequest request, final Map indicesSettings, final Map indicesHealths, @@ -736,286 +747,291 @@ Table buildTable( final String[] indicesToBeQueried, final PaginatedQueryResponse paginatedQueryResponse ) { - final String healthParam = request.param("health"); final Table table = getTableWithHeader(request, paginatedQueryResponse); - if (isActionPaginated() && indicesToBeQueried.length == 0) { - // to handle cases where paginationStrategy couldn't find any indices that should be queried - // and would have returned an empty list of indicesToBeQueried. - return table; - } - indicesSettings.forEach((indexName, settings) -> { if (indicesMetadatas.containsKey(indexName) == false) { // the index exists in the Get Indices response but is not present in the cluster state: // it is likely that the index was deleted in the meanwhile, so we ignore it. return; } + buildRow(indicesSettings, indicesHealths, indicesStats, indicesMetadatas, healthParam, indexName, table); + }); - final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); - final IndexMetadata.State indexState = indexMetadata.getState(); - final IndexStats indexStats = indicesStats.get(indexName); - final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); + return table; + } - final String health; - final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); + protected void buildRow( + final Map indicesSettings, + final Map indicesHealths, + final Map indicesStats, + final Map indicesMetadatas, + final String healthParam, + final String indexName, + Table table + ) { + Settings settings = indicesSettings.get(indexName); + final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); + final IndexMetadata.State indexState = indexMetadata.getState(); + final IndexStats indexStats = indicesStats.get(indexName); + final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); + + final String health; + final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); + if (indexHealth != null) { + health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); + } else if (indexStats != null) { + health = "red*"; + } else { + health = ""; + } + + if (healthParam != null) { + final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); + boolean skip; if (indexHealth != null) { - health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); - } else if (indexStats != null) { - health = "red*"; + // index health is known but does not match the one requested + skip = indexHealth.getStatus() != healthStatusFilter; } else { - health = ""; + // index health is unknown, skip if we don't explicitly request RED health + skip = ClusterHealthStatus.RED != healthStatusFilter; } - - if (healthParam != null) { - final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); - boolean skip; - if (indexHealth != null) { - // index health is known but does not match the one requested - skip = indexHealth.getStatus() != healthStatusFilter; - } else { - // index health is unknown, skip if we don't explicitly request RED health - skip = ClusterHealthStatus.RED != healthStatusFilter; - } - if (skip) { - return; - } + if (skip) { + return; } + } - final CommonStats primaryStats; - final CommonStats totalStats; - - if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { - // TODO: expose docs stats for replicated closed indices - primaryStats = new CommonStats(); - totalStats = new CommonStats(); - } else { - primaryStats = indexStats.getPrimaries(); - totalStats = indexStats.getTotal(); - } - table.startRow(); - table.addCell(health); - table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); - table.addCell(indexName); - table.addCell(indexMetadata.getIndexUUID()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); + final CommonStats primaryStats; + final CommonStats totalStats; - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); + if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { + // TODO: expose docs stats for replicated closed indices + primaryStats = new CommonStats(); + totalStats = new CommonStats(); + } else { + primaryStats = indexStats.getPrimaries(); + totalStats = indexStats.getTotal(); + } + table.startRow(); + table.addCell(health); + table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); + table.addCell(indexName); + table.addCell(indexMetadata.getIndexUUID()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); - table.addCell(indexMetadata.getCreationDate()); - ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); - table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); - table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); - table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + table.addCell(indexMetadata.getCreationDate()); + ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); + table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); - table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); - table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); + table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); + table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); + table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); - table.addCell(totalStats.getTotalMemory()); - table.addCell(primaryStats.getTotalMemory()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); - table.addCell(searchThrottled); + table.addCell(totalStats.getTotalMemory()); + table.addCell(primaryStats.getTotalMemory()); - table.endRow(); - }); + table.addCell(searchThrottled); - return table; + table.endRow(); } @SuppressWarnings("unchecked") diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index 07e6b87ac9aa2..06f8291f749a2 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,6 +9,11 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; +import org.opensearch.action.admin.indices.stats.IndexStats; +import org.opensearch.cluster.health.ClusterIndexHealth; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Table; +import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; @@ -16,6 +21,7 @@ import org.opensearch.rest.pagination.PaginatedQueryResponse; import java.util.List; +import java.util.Map; import java.util.Objects; import static java.util.Arrays.asList; @@ -73,12 +79,33 @@ protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest res } // Next Token in the request will be validated by the IndexStrategyTokenParser itself. if (Objects.nonNull(paginatedQueryRequest.getRequestedTokenStr())) { - IndexBasedPaginationStrategy.IndexStrategyTokenParser.validateIndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); + IndexBasedPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); } return paginatedQueryRequest; } + @Override + protected Table buildTable( + final RestRequest request, + final Map indicesSettings, + final Map indicesHealths, + final Map indicesStats, + final Map indicesMetadatas, + final String[] indicesToBeQueried, + final PaginatedQueryResponse paginatedQueryResponse + ) { + final String healthParam = request.param("health"); + final Table table = getTableWithHeader(request, paginatedQueryResponse); + for (String indexName : indicesToBeQueried) { + if (indicesSettings.containsKey(indexName) == false) { + continue; + } + buildRow(indicesSettings, indicesHealths, indicesStats, indicesMetadatas, healthParam, indexName, table); + } + return table; + } + @Override protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { return new IndexBasedPaginationStrategy(paginatedQueryRequest, PAGINATED_LIST_INDICES_ELEMENT_KEY, clusterStateResponse.getState()); diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java index 8fe06c8fee891..bcb6135b72670 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -10,11 +10,13 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; /** * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. @@ -28,136 +30,167 @@ public class IndexBasedPaginationStrategy implements PaginationStrategy private final List indicesFromRequestedToken; public IndexBasedPaginationStrategy(PaginatedQueryRequest paginatedQueryRequest, String paginatedElement, ClusterState clusterState) { - IndexStrategyTokenParser requestedToken = paginatedQueryRequest.getRequestedTokenStr() == null - ? null - : new IndexStrategyTokenParser(paginatedQueryRequest.getRequestedTokenStr()); - - // Get sorted list of indices not containing the ones created after query start time - List sortedIndicesList = PaginationStrategy.getListOfIndicesSortedByCreateTime( + // Get list of indices metadata sorted by their creation time + List sortedIndicesList = PaginationStrategy.getListOfIndicesSortedByCreateTime( clusterState, - paginatedQueryRequest.getSort(), - requestedToken == null ? Long.MAX_VALUE : requestedToken.queryStartTime + paginatedQueryRequest.getSort() ); - if (sortedIndicesList.isEmpty()) { - // Denotes, that all the indices which were created before the queryStartTime have been deleted. - // No nextToken and indices need to be shown in such cases. - this.indicesFromRequestedToken = new ArrayList<>(); - this.paginatedQueryResponse = new PaginatedQueryResponse(null, paginatedElement); - } else { - final int requestedPageStartIndexNumber = getRequestedPageIndexStartNumber( - requestedToken, - sortedIndicesList, - clusterState, - paginatedQueryRequest.getSort() - ); // inclusive - int requestedPageEndIndexNumber = Math.min( - requestedPageStartIndexNumber + paginatedQueryRequest.getSize(), - sortedIndicesList.size() - ); // exclusive - - this.indicesFromRequestedToken = sortedIndicesList.subList(requestedPageStartIndexNumber, requestedPageEndIndexNumber); - - // Set the queryStart time as the timestamp of latest created index if requested token is null. - long queryStartTime = requestedToken == null - ? DESCENDING_SORT_PARAM_VALUE.equals(paginatedQueryRequest.getSort()) - ? clusterState.metadata().indices().get(sortedIndicesList.get(0)).getCreationDate() - : clusterState.metadata().indices().get(sortedIndicesList.get(sortedIndicesList.size() - 1)).getCreationDate() - : requestedToken.queryStartTime; - - this.paginatedQueryResponse = new PaginatedQueryResponse( - requestedPageEndIndexNumber >= sortedIndicesList.size() - ? null - : new IndexStrategyTokenParser( - requestedPageEndIndexNumber, - clusterState.metadata().indices().get(sortedIndicesList.get(requestedPageEndIndexNumber - 1)).getCreationDate(), - queryStartTime, - sortedIndicesList.get(requestedPageEndIndexNumber - 1) - ).generateEncryptedToken(), - paginatedElement - ); - } + this.indicesFromRequestedToken = getIndicesFromRequestedToken(sortedIndicesList, paginatedQueryRequest); + this.paginatedQueryResponse = getPaginatedResponseFromRequestedToken(sortedIndicesList, paginatedQueryRequest, paginatedElement); } - @Override - @Nullable - public PaginatedQueryResponse getPaginatedQueryResponse() { - return paginatedQueryResponse; + private List getIndicesFromRequestedToken(List sortedIndicesList, PaginatedQueryRequest paginatedQueryRequest) { + if (sortedIndicesList.isEmpty()) { + return new ArrayList<>(); + } + final int requestedPageStartIndexNumber = getRequestedPageIndexStartNumber(paginatedQueryRequest, sortedIndicesList); // inclusive + int requestedPageEndIndexNumber = Math.min( + requestedPageStartIndexNumber + paginatedQueryRequest.getSize(), + sortedIndicesList.size() + ); // exclusive + return sortedIndicesList.subList(requestedPageStartIndexNumber, requestedPageEndIndexNumber) + .stream() + .map(indexMetadata -> indexMetadata.getIndex().getName()) + .collect(Collectors.toList()); } - @Override - public List getElementsFromRequestedToken() { - return Objects.isNull(indicesFromRequestedToken) ? new ArrayList<>() : indicesFromRequestedToken; + private PaginatedQueryResponse getPaginatedResponseFromRequestedToken( + List sortedIndicesList, + PaginatedQueryRequest paginatedQueryRequest, + String paginatedElement + ) { + if (sortedIndicesList.isEmpty()) { + return new PaginatedQueryResponse(null, paginatedElement); + } + int positionToStartNextPage = Math.min( + getRequestedPageIndexStartNumber(paginatedQueryRequest, sortedIndicesList) + paginatedQueryRequest.getSize(), + sortedIndicesList.size() + ); + return new PaginatedQueryResponse( + positionToStartNextPage >= sortedIndicesList.size() + ? null + : new IndexStrategyToken( + positionToStartNextPage, + sortedIndicesList.get(positionToStartNextPage - 1).getCreationDate(), + sortedIndicesList.get(positionToStartNextPage - 1).getIndex().getName() + ).generateEncryptedToken(), + paginatedElement + ); } - private int getRequestedPageIndexStartNumber( - final IndexStrategyTokenParser requestedTokenParser, - final List sortedIndicesList, - final ClusterState clusterState, - final String sortOrder - ) { - if (Objects.isNull(requestedTokenParser)) { + private int getRequestedPageIndexStartNumber(PaginatedQueryRequest paginatedQueryRequest, List sortedIndicesList) { + if (Objects.isNull(paginatedQueryRequest.getRequestedTokenStr())) { + // first paginated query, start from first index. return 0; } - int requestedPageStartIndexNumber = Math.min(requestedTokenParser.posToStartPage, sortedIndicesList.size()); - if (requestedPageStartIndexNumber > 0 - && !Objects.equals(sortedIndicesList.get(requestedPageStartIndexNumber - 1), requestedTokenParser.nameOfLastRespondedIndex)) { - // case denoting an already responded index has been deleted while the paginated queries are being executed + + IndexStrategyToken requestedToken = new IndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); + // If the already requested indices have been deleted, the position to start in the last token could be + // greater than the sorted list's size, hence limiting it to current list's size. + int requestedPageStartIndexNumber = Math.min(requestedToken.posToStartPage, sortedIndicesList.size()); + IndexMetadata currentIndexAtLastSentPosition = sortedIndicesList.get(requestedPageStartIndexNumber - 1); + + if (!Objects.equals(currentIndexAtLastSentPosition.getIndex().getName(), requestedToken.nameOfLastRespondedIndex)) { + // case denoting already responded index/indices has/have been deleted/added in between the paginated queries. // find the index whose creation time is just after/before (based on sortOrder) the index which was last responded. - while (requestedPageStartIndexNumber > 0) { - long creationTimeOfIndex = clusterState.metadata() - .indices() - .get(sortedIndicesList.get(requestedPageStartIndexNumber - 1)) - .getCreationDate(); - if ((!DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - && creationTimeOfIndex < requestedTokenParser.creationTimeOfLastRespondedIndex) - || (DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - && creationTimeOfIndex > requestedTokenParser.creationTimeOfLastRespondedIndex)) { - break; + if (!DESCENDING_SORT_PARAM_VALUE.equals(paginatedQueryRequest.getSort())) { + // For ascending sort order, if indices were deleted, the index to start current page could only have + // moved upwards (at a smaller position) in the sorted list. Traverse backwards to find such index + while (requestedPageStartIndexNumber > 0 + && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) + .getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex)) { + requestedPageStartIndexNumber--; + } + } else { + // For descending order, there could be following 2 possibilities: + // 1. Number of already responded indices which got deleted is greater than newly created ones. + // -> The index to start the page from, would have shifted up in the list. Traverse backwards to find it. + // 2. Number of indices which got created is greater than number of already responded indices which got deleted. + // -> The index to start the page from, would have shifted down in the list. Traverse forward to find it. + boolean traverseForward = currentIndexAtLastSentPosition + .getCreationDate() >= requestedToken.creationTimeOfLastRespondedIndex; + if (traverseForward) { + while (requestedPageStartIndexNumber < sortedIndicesList.size() + && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) + .getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex)) { + requestedPageStartIndexNumber++; + } + } else { + while (requestedPageStartIndexNumber > 0 + && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) + .getCreationDate() < requestedToken.creationTimeOfLastRespondedIndex)) { + requestedPageStartIndexNumber--; + } } - requestedPageStartIndexNumber--; } } return requestedPageStartIndexNumber; } + @Override + @Nullable + public PaginatedQueryResponse getPaginatedQueryResponse() { + return paginatedQueryResponse; + } + + @Override + public List getElementsFromRequestedToken() { + return Objects.isNull(indicesFromRequestedToken) ? new ArrayList<>() : indicesFromRequestedToken; + } + /** * TokenParser to be used by {@link IndexBasedPaginationStrategy}. * Token would like: IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + * QueryStartTime + $ + NameOfLastRespondedIndex */ - public static class IndexStrategyTokenParser { + public static class IndexStrategyToken { + + private static final String TOKEN_JOIN_DELIMITER = "|"; + private static final String TOKEN_SPLIT_REGEX = "\\|"; + private static final int START_PAGE_FIELD_POSITION_IN_TOKEN = 0; + private static final int CREATION_TIME_FIELD_POSITION_IN_TOKEN = 1; + /** + * Denotes the position in the sorted list of indices to start building the page from. + */ private final int posToStartPage; + + /** + * Represents creation times of last index which was displayed in the previous page. + * Used to identify the new start point in case the indices get created/deleted while queries are executed. + */ private final long creationTimeOfLastRespondedIndex; - private final long queryStartTime; + + /** + * Represents name of the last index which was displayed in the previous page. + * Used to identify whether the sorted list of indices has changed or not. + */ private final String nameOfLastRespondedIndex; - public IndexStrategyTokenParser(String requestedTokenString) { + public IndexStrategyToken(String requestedTokenString) { validateIndexStrategyToken(requestedTokenString); String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); - final String[] decryptedTokenElements = decryptedToken.split("\\$"); - this.posToStartPage = Integer.parseInt(decryptedTokenElements[0]); - this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); - this.queryStartTime = Long.parseLong(decryptedTokenElements[2]); - this.nameOfLastRespondedIndex = decryptedTokenElements[3]; + final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); + this.posToStartPage = Integer.parseInt(decryptedTokenElements[START_PAGE_FIELD_POSITION_IN_TOKEN]); + this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATION_TIME_FIELD_POSITION_IN_TOKEN]); + this.nameOfLastRespondedIndex = decryptedTokenElements[2]; } - public IndexStrategyTokenParser( - int indexNumberToStartPageFrom, - long creationTimeOfLastRespondedIndex, - long queryStartTime, - String nameOfLastRespondedIndex - ) { + public IndexStrategyToken(int indexNumberToStartPageFrom, long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); this.posToStartPage = indexNumberToStartPageFrom; this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; - this.queryStartTime = queryStartTime; this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; } public String generateEncryptedToken() { return PaginationStrategy.encryptStringToken( - posToStartPage + "$" + creationTimeOfLastRespondedIndex + "$" + queryStartTime + "$" + nameOfLastRespondedIndex + String.join( + TOKEN_JOIN_DELIMITER, + String.valueOf(posToStartPage), + String.valueOf(creationTimeOfLastRespondedIndex), + nameOfLastRespondedIndex + ) ); } @@ -171,15 +204,14 @@ public String generateEncryptedToken() { public static void validateIndexStrategyToken(String requestedTokenString) { Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); - final String[] decryptedTokenElements = decryptedToken.split("\\$"); - if (decryptedTokenElements.length != 4) { + final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); + if (decryptedTokenElements.length != 3) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } try { int posToStartPage = Integer.parseInt(decryptedTokenElements[0]); long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); - long queryStartTime = Long.parseLong(decryptedTokenElements[2]); - if (posToStartPage < 0 || creationTimeOfLastRespondedIndex < 0 || queryStartTime < 0) { + if (posToStartPage <= 0 || creationTimeOfLastRespondedIndex <= 0) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } } catch (NumberFormatException exception) { diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index e9d25d38eeaa9..58059f6bdc6ba 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -10,6 +10,7 @@ import org.opensearch.OpenSearchParseException; import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; import java.util.Base64; import java.util.List; @@ -43,32 +44,19 @@ public interface PaginationStrategy { /** * - * Utility method to get list of indices sorted by their creation time with {@param latestValidIndexCreateTime} - * being used to filter out the indices created after it. + * Utility method to get list of indices sorted by their creation time. */ - static List getListOfIndicesSortedByCreateTime( - final ClusterState clusterState, - String sortOrder, - final long latestValidIndexCreateTime - ) { - // Filter out the indices which have been created after the latest index which was present when - // paginated query started. Also, sort the indices list based on their creation timestamps - return clusterState.getRoutingTable() - .getIndicesRouting() - .keySet() - .stream() - .filter(index -> (latestValidIndexCreateTime >= clusterState.metadata().indices().get(index).getCreationDate())) - .sorted((index1, index2) -> { - Long index1CreationTimeStamp = clusterState.metadata().indices().get(index1).getCreationDate(); - Long index2CreationTimeStamp = clusterState.metadata().indices().get(index2).getCreationDate(); - if (index1CreationTimeStamp.equals(index2CreationTimeStamp)) { - return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) ? index2.compareTo(index1) : index1.compareTo(index2); - } + static List getListOfIndicesSortedByCreateTime(final ClusterState clusterState, String sortOrder) { + return clusterState.metadata().indices().values().stream().sorted((metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - ? Long.compare(index2CreationTimeStamp, index1CreationTimeStamp) - : Long.compare(index1CreationTimeStamp, index2CreationTimeStamp); - }) - .collect(Collectors.toList()); + ? metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()) + : metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); + } + return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) + ? Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()) + : Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); + }).collect(Collectors.toList()); } static String encryptStringToken(String tokenString) { diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java index b85a1a44d5074..d0a6a8e79973b 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java @@ -18,43 +18,15 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.test.OpenSearchTestCase; +import java.util.List; + import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; public class IndexBasedPaginationStrategyTests extends OpenSearchTestCase { - private final IndexMetadata indexMetadata1 = IndexMetadata.builder("test-index-1") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 1)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) - .build(); - private final IndexRoutingTable.Builder indexRoutingTable1 = new IndexRoutingTable.Builder(indexMetadata1.getIndex()); - private final IndexMetadata indexMetadata2 = IndexMetadata.builder("test-index-2") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 2)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) - .build(); - private final IndexRoutingTable.Builder indexRoutingTable2 = new IndexRoutingTable.Builder(indexMetadata2.getIndex()); - private final IndexMetadata indexMetadata3 = IndexMetadata.builder("test-index-3") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 3)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) - .build(); - private final IndexRoutingTable.Builder indexRoutingTable3 = new IndexRoutingTable.Builder(indexMetadata3.getIndex()); - private final IndexMetadata indexMetadata4 = IndexMetadata.builder("test-index-4") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 4)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) - .build(); - private final IndexRoutingTable.Builder indexRoutingTable4 = new IndexRoutingTable.Builder(indexMetadata4.getIndex()); - private final IndexMetadata indexMetadata5 = IndexMetadata.builder("test-index-5") - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, 5)) - .numberOfShards(between(1, 10)) - .numberOfReplicas(randomInt(20)) - .build(); - private final IndexRoutingTable.Builder indexRoutingTable5 = new IndexRoutingTable.Builder(indexMetadata5.getIndex()); - public void testRetrieveAllIndicesInAscendingOrder() { - ClusterState clusterState = getRandomClusterState(); + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); @@ -81,7 +53,8 @@ public void testRetrieveAllIndicesInAscendingOrder() { } public void testRetrieveAllIndicesInDescendingOrder() { - ClusterState clusterState = getRandomClusterState(); + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); @@ -109,142 +82,66 @@ public void testRetrieveAllIndicesInDescendingOrder() { public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // Query1 with 4 indices in clusterState (test-index1,2,3,4) - ClusterState clusterState = getRandomClusterState(); + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Query2 adding index5 to clusterState - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata2, true) - .put(indexMetadata3, true) - .put(indexMetadata4, true) - .put(indexMetadata5, true) - .build() - ) - .routingTable( - RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable2) - .add(indexRoutingTable3) - .add(indexRoutingTable4) - .add(indexRoutingTable5) - .build() - ) - .build(); + // Adding index5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index2 which has already been displayed, still index3 should get displayed - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata3, true) - .put(indexMetadata4, true) - .put(indexMetadata5, true) - .build() - ) - .routingTable( - RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable3) - .add(indexRoutingTable4) - .add(indexRoutingTable5) - .build() - ) - .build(); + // Deleting test-index-2 which has already been displayed, still test-index-2 should get displayed + clusterState = deleteIndexFromClusterState(clusterState, 2); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index4 which is not yet displayed which otherwise should have been displayed in the following query - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata(Metadata.builder().put(indexMetadata1, true).put(indexMetadata3, true).put(indexMetadata5, true).build()) - .routingTable(RoutingTable.builder().add(indexRoutingTable1).add(indexRoutingTable3).add(indexRoutingTable5).build()) - .build(); + // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query + // instead test-index-5 should now get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 4); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); } public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDescOrder() { - // Query1 with 4 indices in clusterState (test-index1,2,3,4) - ClusterState clusterState = getRandomClusterState(); + // Query1 with 4 indices in clusterState (test-index1,2,3,4). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Query2 adding index5 to clusterState - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata2, true) - .put(indexMetadata3, true) - .put(indexMetadata4, true) - .put(indexMetadata5, true) - .build() - ) - .routingTable( - RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable2) - .add(indexRoutingTable3) - .add(indexRoutingTable4) - .add(indexRoutingTable5) - .build() - ) - .build(); + // adding test-index-5 to clusterState, before executing next query. + clusterState = addIndexToClusterState(clusterState, 5); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index3 which has already been displayed, still index2 should get displayed - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata2, true) - .put(indexMetadata4, true) - .put(indexMetadata5, true) - .build() - ) - .routingTable( - RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable2) - .add(indexRoutingTable4) - .add(indexRoutingTable5) - .build() - ) - .build(); + // Deleting test-index-3 which has already been displayed, still index2 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 3); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index1 which is not yet displayed which otherwise should have been displayed in the following query - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata(Metadata.builder().put(indexMetadata2, true).put(indexMetadata4, true).put(indexMetadata5, true).build()) - .routingTable(RoutingTable.builder().add(indexRoutingTable2).add(indexRoutingTable4).add(indexRoutingTable5).build()) - .build(); + // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. + clusterState = deleteIndexFromClusterState(clusterState, 1); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); @@ -252,52 +149,32 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDe } public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() { - // Query1 with 5 indices in clusterState (test-index1,2,3,4,5) - ClusterState clusterState = ClusterState.builder(new ClusterName("test")) - .metadata( - Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata2, true) - .put(indexMetadata3, true) - .put(indexMetadata4, true) - .put(indexMetadata5, true) - .build() - ) - .routingTable( - RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable2) - .add(indexRoutingTable3) - .add(indexRoutingTable4) - .add(indexRoutingTable5) - .build() - ) - .build(); + // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Query2, no changes to clusterState + // executing next query without any changes to clusterState paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index1,index2 & index3. index4 should get displayed - clusterState = ClusterState.builder(new ClusterName("test")) - .metadata(Metadata.builder().put(indexMetadata4, true).put(indexMetadata5, true).build()) - .routingTable(RoutingTable.builder().add(indexRoutingTable4).add(indexRoutingTable5).build()) - .build(); + // Deleting test-index-1, test-index-2 & test-index-3 and executing next query. test-index-4 should get displayed. + clusterState = deleteIndexFromClusterState(clusterState, 1); + clusterState = deleteIndexFromClusterState(clusterState, 2); + clusterState = deleteIndexFromClusterState(clusterState, 3); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - // Deleting index1 which is not yet displayed which otherwise should have been displayed in the following query + // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); @@ -307,7 +184,7 @@ public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { try { - new IndexBasedPaginationStrategy.IndexStrategyTokenParser(null); + new IndexBasedPaginationStrategy.IndexStrategyToken(null); fail("expected exception"); } catch (Exception e) { assert e.getMessage().contains("requestedTokenString can not be null"); @@ -315,72 +192,73 @@ public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { } public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { - try { - // % is not allowed in base64 encoding - new IndexBasedPaginationStrategy.IndexStrategyTokenParser("3%4%5"); - fail("expected exception"); - } catch (OpenSearchParseException e) { - assert e.getMessage().contains("[next_token] has been tainted"); - } + assertThrows(OpenSearchParseException.class, () -> new IndexBasedPaginationStrategy.IndexStrategyToken("3%4%5")); } - public void testIndexStrategyPageTokenWithLessElementsInRequestedToken() { - String encryptedRequestToken = PaginationStrategy.encryptStringToken("1$1725361543$1725361543"); - try { - // should throw exception as it expects 4 elements separated by $ - new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); - fail("expected exception"); - } catch (OpenSearchParseException e) { - assert e.getMessage().contains("[next_token] has been tainted"); - } + public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { + assertThrows( + OpenSearchParseException.class, + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$1725361543")) + ); + assertThrows( + OpenSearchParseException.class, + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$1725361543$index$12345")) + ); } public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { - { - String encryptedRequestToken = PaginationStrategy.encryptStringToken("1$-1725361543$-1725361543$index"); - try { - // should not accept invalid long values (negative) - new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); - fail("expected exception"); - } catch (OpenSearchParseException e) { - assert e.getMessage().contains("[next_token] has been tainted"); - } - } - - { - String encryptedRequestToken = PaginationStrategy.encryptStringToken("-1$1725361543$1725361543$index"); - try { - // should not accept invalid int values (negative) - new IndexBasedPaginationStrategy.IndexStrategyTokenParser(encryptedRequestToken); - fail("expected exception"); - } catch (OpenSearchParseException e) { - assert e.getMessage().contains("[next_token] has been tainted"); - } - } + assertThrows( + OpenSearchParseException.class, + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$-1725361543$index")) + ); + assertThrows( + OpenSearchParseException.class, + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1$1725361543$index")) + ); } public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { try { - new IndexBasedPaginationStrategy.IndexStrategyTokenParser(1, 1234l, 1234l, null); + new IndexBasedPaginationStrategy.IndexStrategyToken(1, 1234l, null); fail("expected exception"); } catch (Exception e) { assert e.getMessage().contains("index name should be provided"); } } - private ClusterState getRandomClusterState() { - Metadata metadata = Metadata.builder() - .put(indexMetadata1, true) - .put(indexMetadata2, true) - .put(indexMetadata3, true) - .put(indexMetadata4, true) + /** + * @param indexNumbers would be used to create indices having names with integer appended after foo, like foo1, foo2. + * @return random clusterState consisting of indices having their creation times set to the integer used to name them. + */ + private ClusterState getRandomClusterState(List indexNumbers) { + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().build()) + .routingTable(RoutingTable.builder().build()) .build(); - RoutingTable routingTable = RoutingTable.builder() - .add(indexRoutingTable1) - .add(indexRoutingTable2) - .add(indexRoutingTable3) - .add(indexRoutingTable4) + for (Integer indexNumber : indexNumbers) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + return clusterState; + } + + private ClusterState addIndexToClusterState(ClusterState clusterState, int indexNumber) { + IndexMetadata indexMetadata = IndexMetadata.builder("test-index-" + indexNumber) + .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, indexNumber)) + .numberOfShards(between(1, 10)) + .numberOfReplicas(randomInt(20)) + .build(); + IndexRoutingTable.Builder indexRoutingTableBuilder = new IndexRoutingTable.Builder(indexMetadata.getIndex()); + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).put(indexMetadata, true).build()) + .routingTable(RoutingTable.builder(clusterState.routingTable()).add(indexRoutingTableBuilder).build()) .build(); - return ClusterState.builder(new ClusterName("test")).metadata(metadata).routingTable(routingTable).build(); } + + private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int indexNumber) { + return ClusterState.builder(clusterState) + .metadata(Metadata.builder(clusterState.metadata()).remove("test-index-" + indexNumber)) + .routingTable(RoutingTable.builder(clusterState.routingTable()).remove("test-index-" + indexNumber).build()) + .build(); + } + } From 3bedc8244bfdc981f06a61909b8e4cb63954c683 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Thu, 19 Sep 2024 10:11:40 +0530 Subject: [PATCH 06/12] Refactoring strategy to simplify generating list of requested indices Signed-off-by: Harsh Garg --- .../action/list/RestIndicesListAction.java | 3 +- .../IndexBasedPaginationStrategy.java | 178 +++++++----------- .../pagination/PaginatedQueryRequest.java | 8 +- .../rest/pagination/PaginationStrategy.java | 22 +-- .../IndexBasedPaginationStrategyTests.java | 163 +++++++++------- 5 files changed, 180 insertions(+), 194 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index 06f8291f749a2..a6c6775b0f47a 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -37,7 +37,6 @@ public class RestIndicesListAction extends RestIndicesAction { protected static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING = 1000; protected static final int DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = 1000; - protected static final String PAGINATED_LIST_INDICES_ELEMENT_KEY = "indices"; @Override public List routes() { @@ -108,7 +107,7 @@ protected Table buildTable( @Override protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { - return new IndexBasedPaginationStrategy(paginatedQueryRequest, PAGINATED_LIST_INDICES_ELEMENT_KEY, clusterStateResponse.getState()); + return new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterStateResponse.getState()); } @Override diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java index bcb6135b72670..02c08a7a46a3b 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java @@ -14,10 +14,14 @@ import org.opensearch.common.Nullable; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_ASCENDING_SORT; + /** * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. * The strategy considers create timestamps of indices as the keys to iterate over pages. @@ -29,102 +33,79 @@ public class IndexBasedPaginationStrategy implements PaginationStrategy private final PaginatedQueryResponse paginatedQueryResponse; private final List indicesFromRequestedToken; - public IndexBasedPaginationStrategy(PaginatedQueryRequest paginatedQueryRequest, String paginatedElement, ClusterState clusterState) { - // Get list of indices metadata sorted by their creation time - List sortedIndicesList = PaginationStrategy.getListOfIndicesSortedByCreateTime( + private static final String DEFAULT_INDICES_PAGINATED_ELEMENT = "indices"; + + public IndexBasedPaginationStrategy(PaginatedQueryRequest paginatedQueryRequest, ClusterState clusterState) { + // Get list of indices metadata sorted by their creation time and filtered by the last send index + List sortedIndicesList = PaginationStrategy.getSortedIndexMetadata( clusterState, - paginatedQueryRequest.getSort() + getMetadataListFilter(paginatedQueryRequest.getRequestedTokenStr(), paginatedQueryRequest.getSort()), + getMetadataListComparator(paginatedQueryRequest.getSort()) ); - this.indicesFromRequestedToken = getIndicesFromRequestedToken(sortedIndicesList, paginatedQueryRequest); - this.paginatedQueryResponse = getPaginatedResponseFromRequestedToken(sortedIndicesList, paginatedQueryRequest, paginatedElement); + List metadataListForRequestedToken = getMetadataListForRequestedToken(sortedIndicesList, paginatedQueryRequest); + this.indicesFromRequestedToken = metadataListForRequestedToken.stream() + .map(metadata -> metadata.getIndex().getName()) + .collect(Collectors.toList()); + this.paginatedQueryResponse = getPaginatedResponseForRequestedToken(paginatedQueryRequest.getSize(), sortedIndicesList); } - private List getIndicesFromRequestedToken(List sortedIndicesList, PaginatedQueryRequest paginatedQueryRequest) { - if (sortedIndicesList.isEmpty()) { - return new ArrayList<>(); + private static Predicate getMetadataListFilter(String requestedTokenStr, String sortOrder) { + boolean isAscendingSort = sortOrder.equals(PAGINATED_QUERY_ASCENDING_SORT); + IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() + ? null + : new IndexStrategyToken(requestedTokenStr); + if (Objects.isNull(requestedToken)) { + return indexMetadata -> true; } - final int requestedPageStartIndexNumber = getRequestedPageIndexStartNumber(paginatedQueryRequest, sortedIndicesList); // inclusive - int requestedPageEndIndexNumber = Math.min( - requestedPageStartIndexNumber + paginatedQueryRequest.getSize(), - sortedIndicesList.size() - ); // exclusive - return sortedIndicesList.subList(requestedPageStartIndexNumber, requestedPageEndIndexNumber) - .stream() - .map(indexMetadata -> indexMetadata.getIndex().getName()) - .collect(Collectors.toList()); + return indexMetadata -> { + if (indexMetadata.getIndex().getName().equals(requestedToken.nameOfLastRespondedIndex)) { + return false; + } else if (indexMetadata.getCreationDate() == requestedToken.creationTimeOfLastRespondedIndex) { + return isAscendingSort + ? indexMetadata.getIndex().getName().compareTo(requestedToken.nameOfLastRespondedIndex) > 0 + : indexMetadata.getIndex().getName().compareTo(requestedToken.nameOfLastRespondedIndex) < 0; + } + return isAscendingSort + ? indexMetadata.getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex + : indexMetadata.getCreationDate() < requestedToken.creationTimeOfLastRespondedIndex; + }; + } + + private static Comparator getMetadataListComparator(String sortOrder) { + boolean isAscendingSort = sortOrder.equals(PAGINATED_QUERY_ASCENDING_SORT); + return (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return isAscendingSort + ? metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()) + : metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); + } + return isAscendingSort + ? Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()) + : Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()); + }; } - private PaginatedQueryResponse getPaginatedResponseFromRequestedToken( + private List getMetadataListForRequestedToken( List sortedIndicesList, - PaginatedQueryRequest paginatedQueryRequest, - String paginatedElement + PaginatedQueryRequest paginatedQueryRequest ) { if (sortedIndicesList.isEmpty()) { - return new PaginatedQueryResponse(null, paginatedElement); + return new ArrayList<>(); } - int positionToStartNextPage = Math.min( - getRequestedPageIndexStartNumber(paginatedQueryRequest, sortedIndicesList) + paginatedQueryRequest.getSize(), - sortedIndicesList.size() - ); - return new PaginatedQueryResponse( - positionToStartNextPage >= sortedIndicesList.size() - ? null - : new IndexStrategyToken( - positionToStartNextPage, - sortedIndicesList.get(positionToStartNextPage - 1).getCreationDate(), - sortedIndicesList.get(positionToStartNextPage - 1).getIndex().getName() - ).generateEncryptedToken(), - paginatedElement - ); + return sortedIndicesList.subList(0, Math.min(paginatedQueryRequest.getSize(), sortedIndicesList.size())); } - private int getRequestedPageIndexStartNumber(PaginatedQueryRequest paginatedQueryRequest, List sortedIndicesList) { - if (Objects.isNull(paginatedQueryRequest.getRequestedTokenStr())) { - // first paginated query, start from first index. - return 0; - } - - IndexStrategyToken requestedToken = new IndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); - // If the already requested indices have been deleted, the position to start in the last token could be - // greater than the sorted list's size, hence limiting it to current list's size. - int requestedPageStartIndexNumber = Math.min(requestedToken.posToStartPage, sortedIndicesList.size()); - IndexMetadata currentIndexAtLastSentPosition = sortedIndicesList.get(requestedPageStartIndexNumber - 1); - - if (!Objects.equals(currentIndexAtLastSentPosition.getIndex().getName(), requestedToken.nameOfLastRespondedIndex)) { - // case denoting already responded index/indices has/have been deleted/added in between the paginated queries. - // find the index whose creation time is just after/before (based on sortOrder) the index which was last responded. - if (!DESCENDING_SORT_PARAM_VALUE.equals(paginatedQueryRequest.getSort())) { - // For ascending sort order, if indices were deleted, the index to start current page could only have - // moved upwards (at a smaller position) in the sorted list. Traverse backwards to find such index - while (requestedPageStartIndexNumber > 0 - && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) - .getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex)) { - requestedPageStartIndexNumber--; - } - } else { - // For descending order, there could be following 2 possibilities: - // 1. Number of already responded indices which got deleted is greater than newly created ones. - // -> The index to start the page from, would have shifted up in the list. Traverse backwards to find it. - // 2. Number of indices which got created is greater than number of already responded indices which got deleted. - // -> The index to start the page from, would have shifted down in the list. Traverse forward to find it. - boolean traverseForward = currentIndexAtLastSentPosition - .getCreationDate() >= requestedToken.creationTimeOfLastRespondedIndex; - if (traverseForward) { - while (requestedPageStartIndexNumber < sortedIndicesList.size() - && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) - .getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex)) { - requestedPageStartIndexNumber++; - } - } else { - while (requestedPageStartIndexNumber > 0 - && (sortedIndicesList.get(requestedPageStartIndexNumber - 1) - .getCreationDate() < requestedToken.creationTimeOfLastRespondedIndex)) { - requestedPageStartIndexNumber--; - } - } - } + private PaginatedQueryResponse getPaginatedResponseForRequestedToken(int pageSize, List sortedIndicesList) { + if (sortedIndicesList.size() <= pageSize) { + return new PaginatedQueryResponse(null, DEFAULT_INDICES_PAGINATED_ELEMENT); } - return requestedPageStartIndexNumber; + return new PaginatedQueryResponse( + new IndexStrategyToken( + sortedIndicesList.get(pageSize - 1).getCreationDate(), + sortedIndicesList.get(pageSize - 1).getIndex().getName() + ).generateEncryptedToken(), + DEFAULT_INDICES_PAGINATED_ELEMENT + ); } @Override @@ -140,20 +121,15 @@ public List getElementsFromRequestedToken() { /** * TokenParser to be used by {@link IndexBasedPaginationStrategy}. - * Token would like: IndexNumberToStartTheNextPageFrom + $ + CreationTimeOfLastRespondedIndex + $ + - * QueryStartTime + $ + NameOfLastRespondedIndex + * Token would like: IndexNumberToStartTheNextPageFrom + | + CreationTimeOfLastRespondedIndex + | + + * QueryStartTime + | + NameOfLastRespondedIndex */ public static class IndexStrategyToken { private static final String TOKEN_JOIN_DELIMITER = "|"; private static final String TOKEN_SPLIT_REGEX = "\\|"; - private static final int START_PAGE_FIELD_POSITION_IN_TOKEN = 0; - private static final int CREATION_TIME_FIELD_POSITION_IN_TOKEN = 1; - - /** - * Denotes the position in the sorted list of indices to start building the page from. - */ - private final int posToStartPage; + private static final int CREATION_TIME_FIELD_POSITION_IN_TOKEN = 0; + private static final int INDEX_NAME_FIELD_POSITION_IN_TOKEN = 1; /** * Represents creation times of last index which was displayed in the previous page. @@ -171,26 +147,19 @@ public IndexStrategyToken(String requestedTokenString) { validateIndexStrategyToken(requestedTokenString); String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); - this.posToStartPage = Integer.parseInt(decryptedTokenElements[START_PAGE_FIELD_POSITION_IN_TOKEN]); this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATION_TIME_FIELD_POSITION_IN_TOKEN]); - this.nameOfLastRespondedIndex = decryptedTokenElements[2]; + this.nameOfLastRespondedIndex = decryptedTokenElements[INDEX_NAME_FIELD_POSITION_IN_TOKEN]; } - public IndexStrategyToken(int indexNumberToStartPageFrom, long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { + public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); - this.posToStartPage = indexNumberToStartPageFrom; this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; } public String generateEncryptedToken() { return PaginationStrategy.encryptStringToken( - String.join( - TOKEN_JOIN_DELIMITER, - String.valueOf(posToStartPage), - String.valueOf(creationTimeOfLastRespondedIndex), - nameOfLastRespondedIndex - ) + String.join(TOKEN_JOIN_DELIMITER, String.valueOf(creationTimeOfLastRespondedIndex), nameOfLastRespondedIndex) ); } @@ -205,13 +174,12 @@ public static void validateIndexStrategyToken(String requestedTokenString) { Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); - if (decryptedTokenElements.length != 3) { + if (decryptedTokenElements.length != 2) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } try { - int posToStartPage = Integer.parseInt(decryptedTokenElements[0]); - long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[1]); - if (posToStartPage <= 0 || creationTimeOfLastRespondedIndex <= 0) { + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATION_TIME_FIELD_POSITION_IN_TOKEN]); + if (creationTimeOfLastRespondedIndex <= 0) { throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); } } catch (NumberFormatException exception) { diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java index cfcf93d6ce95c..eb18429676c1c 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java @@ -14,18 +14,20 @@ * * Class specific to paginated queries, which will contain common query params required by a paginated API. */ -@PublicApi(since = "2.17.0") +@PublicApi(since = "3.0.0") public class PaginatedQueryRequest { public static final String PAGINATED_QUERY_PARAM_SORT_KEY = "sort"; public static final String PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY = "next_token"; public static final String PAGINATED_QUERY_PARAM_SIZE_KEY = "size"; + public static final String PAGINATED_QUERY_ASCENDING_SORT = "ascending"; + private final String requestedTokenStr; private final String sort; private final int size; - public PaginatedQueryRequest(String requested_token, String sort, int size) { - this.requestedTokenStr = requested_token; + public PaginatedQueryRequest(String requestedToken, String sort, int size) { + this.requestedTokenStr = requestedToken; this.sort = sort; this.size = size; } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index 58059f6bdc6ba..8675b85236755 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -13,8 +13,10 @@ import org.opensearch.cluster.metadata.IndexMetadata; import java.util.Base64; +import java.util.Comparator; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; @@ -26,7 +28,6 @@ */ public interface PaginationStrategy { - String DESCENDING_SORT_PARAM_VALUE = "descending"; String INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE = "Parameter [next_token] has been tainted and is incorrect. Please provide a valid [next_token]."; @@ -44,19 +45,14 @@ public interface PaginationStrategy { /** * - * Utility method to get list of indices sorted by their creation time. + * Utility method to get list of indices filtered as per {@param filterPredicate} and the sorted according to {@param comparator}. */ - static List getListOfIndicesSortedByCreateTime(final ClusterState clusterState, String sortOrder) { - return clusterState.metadata().indices().values().stream().sorted((metadata1, metadata2) -> { - if (metadata1.getCreationDate() == metadata2.getCreationDate()) { - return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - ? metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()) - : metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); - } - return DESCENDING_SORT_PARAM_VALUE.equals(sortOrder) - ? Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()) - : Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); - }).collect(Collectors.toList()); + static List getSortedIndexMetadata( + final ClusterState clusterState, + Predicate filterPredicate, + Comparator comparator + ) { + return clusterState.metadata().indices().values().stream().filter(filterPredicate).sorted(comparator).collect(Collectors.toList()); } static String encryptStringToken(String tokenString) { diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java index d0a6a8e79973b..8def1420d86c4 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java @@ -18,73 +18,96 @@ import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.test.OpenSearchTestCase; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; public class IndexBasedPaginationStrategyTests extends OpenSearchTestCase { public void testRetrieveAllIndicesInAscendingOrder() { - ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + List indexNumberList = new ArrayList<>(); + final int totalIndices = 100; + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + // creating a cluster state with 100 indices + Collections.shuffle(indexNumberList, getRandom()); + ClusterState clusterState = getRandomClusterState(indexNumberList); + + // Checking pagination response for different pageSizes, which has a mix of even and odd numbers + // to ensure number of indices in last page is not always equal to pageSize. + List pageSizeList = List.of(1, 6, 10, 13); + for (int pageSize : pageSizeList) { + String requestedToken = null; + int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); + for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(requestedToken, "ascending", pageSize); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); + if (pageNumber < totalPagesToFetch) { + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } else { + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } + requestedToken = paginationStrategy.getPaginatedQueryResponse().getNextToken(); + // Asserting all the indices received + int responseItr = 0; + for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(100, pageNumber * pageSize); indexNumber++) { + assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getElementsFromRequestedToken().get(responseItr)); + responseItr++; + } + assertEquals(responseItr, paginationStrategy.getElementsFromRequestedToken().size()); + } + } } public void testRetrieveAllIndicesInDescendingOrder() { - ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); - - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + List indexNumberList = new ArrayList<>(); + final int totalIndices = 100; + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + // creating a cluster state with 100 indices + Collections.shuffle(indexNumberList, getRandom()); + ClusterState clusterState = getRandomClusterState(indexNumberList); + + // Checking pagination response for different pageSizes, which has a mix of even and odd numbers + // to ensure number of indices in last page is not always equal to pageSize. + List pageSizeList = List.of(1, 6, 10, 13); + for (int pageSize : pageSizeList) { + String requestedToken = null; + int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); + int startIndexNumber = totalIndices; + for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { + PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(requestedToken, "descending", pageSize); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); + if (pageNumber < totalPagesToFetch) { + assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } else { + assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + } + requestedToken = paginationStrategy.getPaginatedQueryResponse().getNextToken(); + // Asserting all the indices received + int responseItr = 0; + int endIndexNumberForPage = Math.max(startIndexNumber - pageSize, 0); + for (; startIndexNumber > endIndexNumberForPage; startIndexNumber--) { + assertEquals("test-index-" + startIndexNumber, paginationStrategy.getElementsFromRequestedToken().get(responseItr)); + responseItr++; + } + assertEquals(responseItr, paginationStrategy.getElementsFromRequestedToken().size()); + } + } } public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // Query1 with 4 indices in clusterState (test-index1,2,3,4) ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -92,7 +115,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // Adding index5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -100,7 +123,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // Deleting test-index-2 which has already been displayed, still test-index-2 should get displayed clusterState = deleteIndexFromClusterState(clusterState, 2); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -109,7 +132,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // instead test-index-5 should now get displayed. clusterState = deleteIndexFromClusterState(clusterState, 4); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -119,7 +142,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDe // Query1 with 4 indices in clusterState (test-index1,2,3,4). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -127,7 +150,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDe // adding test-index-5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -135,7 +158,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDe // Deleting test-index-3 which has already been displayed, still index2 should get displayed. clusterState = deleteIndexFromClusterState(clusterState, 3); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -143,7 +166,7 @@ public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDe // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. clusterState = deleteIndexFromClusterState(clusterState, 1); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); } @@ -152,14 +175,14 @@ public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); // executing next query without any changes to clusterState paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -169,14 +192,14 @@ public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() clusterState = deleteIndexFromClusterState(clusterState, 2); clusterState = deleteIndexFromClusterState(clusterState, 3); paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, "test", clusterState); + paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); @@ -198,28 +221,24 @@ public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$1725361543")) + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543")) ); assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$1725361543$index$12345")) + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) ); } public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1$-1725361543$index")) - ); - assertThrows( - OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1$1725361543$index")) + () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index")) ); } public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { try { - new IndexBasedPaginationStrategy.IndexStrategyToken(1, 1234l, null); + new IndexBasedPaginationStrategy.IndexStrategyToken(1234l, null); fail("expected exception"); } catch (Exception e) { assert e.getMessage().contains("index name should be provided"); @@ -243,7 +262,9 @@ private ClusterState getRandomClusterState(List indexNumbers) { private ClusterState addIndexToClusterState(ClusterState clusterState, int indexNumber) { IndexMetadata indexMetadata = IndexMetadata.builder("test-index-" + indexNumber) - .settings(settings(Version.CURRENT).put(SETTING_CREATION_DATE, indexNumber)) + .settings( + settings(Version.CURRENT).put(SETTING_CREATION_DATE, Instant.now().plus(indexNumber, ChronoUnit.SECONDS).toEpochMilli()) + ) .numberOfShards(between(1, 10)) .numberOfReplicas(randomInt(20)) .build(); From 9aebdce81737a53cbcf98cfb008e5df9ba128f52 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Fri, 20 Sep 2024 09:42:36 +0530 Subject: [PATCH 07/12] Adding AbstractListAction and renaming pagination related classes Signed-off-by: Harsh Garg --- .../example/resthandler/ExampleCatAction.java | 2 +- .../org/opensearch/action/ActionModule.java | 7 +- .../java/org/opensearch/common/Table.java | 12 +- .../java/org/opensearch/rest/RestRequest.java | 16 +- .../rest/action/cat/AbstractCatAction.java | 29 +-- .../rest/action/cat/RestAliasAction.java | 2 +- .../rest/action/cat/RestAllocationAction.java | 2 +- .../action/cat/RestCatRecoveryAction.java | 2 +- .../cat/RestCatSegmentReplicationAction.java | 2 +- .../action/cat/RestClusterManagerAction.java | 2 +- .../rest/action/cat/RestCountAction.java | 2 +- .../rest/action/cat/RestFielddataAction.java | 2 +- .../rest/action/cat/RestHealthAction.java | 2 +- .../rest/action/cat/RestIndicesAction.java | 31 +-- .../rest/action/cat/RestNodeAttrsAction.java | 2 +- .../rest/action/cat/RestNodesAction.java | 2 +- .../cat/RestPendingClusterTasksAction.java | 2 +- .../action/cat/RestPitSegmentsAction.java | 2 +- .../rest/action/cat/RestPluginsAction.java | 2 +- .../action/cat/RestRepositoriesAction.java | 2 +- .../rest/action/cat/RestSegmentsAction.java | 2 +- .../rest/action/cat/RestShardsAction.java | 2 +- .../opensearch/rest/action/cat/RestTable.java | 16 +- .../rest/action/cat/RestThreadPoolAction.java | 2 +- .../rest/action/list/AbstractListAction.java | 60 ++++++ .../action/list/RestIndicesListAction.java | 48 +++-- .../rest/action/list/RestListAction.java | 5 +- .../IndexBasedPaginationStrategy.java | 191 ------------------ .../pagination/IndexPaginationStrategy.java | 185 +++++++++++++++++ ...natedQueryRequest.java => PageParams.java} | 15 +- ...natedQueryResponse.java => PageToken.java} | 12 +- .../rest/pagination/PaginationStrategy.java | 4 +- ...java => IndexPaginationStrategyTests.java} | 160 +++++++-------- 33 files changed, 425 insertions(+), 402 deletions(-) create mode 100644 server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java delete mode 100644 server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java create mode 100644 server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java rename server/src/main/java/org/opensearch/rest/pagination/{PaginatedQueryRequest.java => PageParams.java} (63%) rename server/src/main/java/org/opensearch/rest/pagination/{PaginatedQueryResponse.java => PageToken.java} (76%) rename server/src/test/java/org/opensearch/rest/pagination/{IndexBasedPaginationStrategyTests.java => IndexPaginationStrategyTests.java} (54%) diff --git a/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java b/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java index a1a16846988a6..06e0cee69e68f 100644 --- a/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java +++ b/plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java @@ -83,7 +83,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append(documentation()); } diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 9e31189ea8b5e..eb853e615a89c 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -459,6 +459,7 @@ import org.opensearch.rest.action.ingest.RestGetPipelineAction; import org.opensearch.rest.action.ingest.RestPutPipelineAction; import org.opensearch.rest.action.ingest.RestSimulatePipelineAction; +import org.opensearch.rest.action.list.AbstractListAction; import org.opensearch.rest.action.list.RestIndicesListAction; import org.opensearch.rest.action.list.RestListAction; import org.opensearch.rest.action.search.RestClearScrollAction; @@ -801,11 +802,11 @@ private ActionFilters setupActionFilters(List actionPlugins) { public void initRestHandlers(Supplier nodesInCluster) { List catActions = new ArrayList<>(); - List listActions = new ArrayList<>(); + List listActions = new ArrayList<>(); Consumer registerHandler = handler -> { if (handler instanceof AbstractCatAction) { - if (((AbstractCatAction) handler).isActionPaginated()) { - listActions.add((AbstractCatAction) handler); + if (handler instanceof AbstractListAction && ((AbstractListAction) handler).isActionPaginated()) { + listActions.add((AbstractListAction) handler); } else { catActions.add((AbstractCatAction) handler); } diff --git a/server/src/main/java/org/opensearch/common/Table.java b/server/src/main/java/org/opensearch/common/Table.java index 1ba238f8de45b..133ec3052e6c9 100644 --- a/server/src/main/java/org/opensearch/common/Table.java +++ b/server/src/main/java/org/opensearch/common/Table.java @@ -34,7 +34,7 @@ import org.opensearch.common.time.DateFormatter; import org.opensearch.core.common.Strings; -import org.opensearch.rest.pagination.PaginatedQueryResponse; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -63,14 +63,14 @@ public class Table { /** * paginatedQueryResponse if null will imply the Table response is not paginated. */ - private PaginatedQueryResponse paginatedQueryResponse; + private PageToken pageToken; public static final String EPOCH = "epoch"; public static final String TIMESTAMP = "timestamp"; public Table() {} - public Table(@Nullable PaginatedQueryResponse paginatedQueryResponse) { - this.paginatedQueryResponse = paginatedQueryResponse; + public Table(@Nullable PageToken pageToken) { + this.pageToken = pageToken; } public Table startHeaders() { @@ -241,8 +241,8 @@ public Map getAliasMap() { return headerAliasMap; } - public PaginatedQueryResponse getPaginatedQueryResponse() { - return paginatedQueryResponse; + public PageToken getPageToken() { + return pageToken; } /** diff --git a/server/src/main/java/org/opensearch/rest/RestRequest.java b/server/src/main/java/org/opensearch/rest/RestRequest.java index 1b9f5708a00fe..f241b567c3204 100644 --- a/server/src/main/java/org/opensearch/rest/RestRequest.java +++ b/server/src/main/java/org/opensearch/rest/RestRequest.java @@ -51,7 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpChannel; import org.opensearch.http.HttpRequest; -import org.opensearch.rest.pagination.PaginatedQueryRequest; +import org.opensearch.rest.pagination.PageParams; import java.io.IOException; import java.io.InputStream; @@ -68,9 +68,9 @@ import static org.opensearch.common.unit.TimeValue.parseTimeValue; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; -import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY; -import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SIZE_KEY; -import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SORT_KEY; +import static org.opensearch.rest.pagination.PageParams.PARAM_NEXT_TOKEN; +import static org.opensearch.rest.pagination.PageParams.PARAM_SIZE; +import static org.opensearch.rest.pagination.PageParams.PARAM_SORT; /** * REST Request @@ -595,12 +595,8 @@ public static MediaType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } - public PaginatedQueryRequest parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) { - return new PaginatedQueryRequest( - param(PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY), - param(PAGINATED_QUERY_PARAM_SORT_KEY, defaultSortOrder), - paramAsInt(PAGINATED_QUERY_PARAM_SIZE_KEY, defaultPageSize) - ); + public PageParams parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) { + return new PageParams(param(PARAM_NEXT_TOKEN), param(PARAM_SORT, defaultSortOrder), paramAsInt(PARAM_SIZE, defaultPageSize)); } /** diff --git a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java index c85a458d0f0c6..6f4e060363bfb 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java @@ -40,13 +40,11 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.pagination.PaginatedQueryRequest; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Objects; import java.util.Set; import static org.opensearch.rest.action.cat.RestTable.buildHelpWidths; @@ -59,11 +57,9 @@ */ public abstract class AbstractCatAction extends BaseRestHandler { - protected PaginatedQueryRequest paginatedQueryRequest; - protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client); - public abstract void documentation(StringBuilder sb); + protected abstract void documentation(StringBuilder sb); protected abstract Table getTableWithHeader(RestRequest request); @@ -89,10 +85,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes())); }; } else { - if (isActionPaginated()) { - this.paginatedQueryRequest = validateAndGetPaginationMetadata(request); - assert Objects.nonNull(paginatedQueryRequest) : "paginatedQueryRequest can not be null for paginated queries"; - } return doCatRequest(request, client); } } @@ -106,23 +98,4 @@ protected Set responseParams() { return RESPONSE_PARAMS; } - /** - * - * @return boolean denoting whether the RestAction will output paginated responses or not. - * Is kept false by default, every paginated action to override and return true. - */ - public boolean isActionPaginated() { - return false; - } - - /** - * - * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide - * its own implementation. Query params supported by the action specific to pagination along with the respective validations, - * should be added here. - */ - protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest restRequest) { - return null; - } - } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java index a1cf1ca57c04f..4600dddbb361d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java @@ -89,7 +89,7 @@ public RestResponse buildResponse(GetAliasesResponse response) throws Exception } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/aliases\n"); sb.append("/_cat/aliases/{alias}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java index 2aa2b0aaad893..07b0fbbe4a911 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java @@ -78,7 +78,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/allocation\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java index 9e9b8c2e36b5b..26efd9929afea 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java @@ -77,7 +77,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/recovery\n"); sb.append("/_cat/recovery/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java index 2f76a68e7221e..aa325443ba6c9 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java @@ -58,7 +58,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/segment_replication\n"); sb.append("/_cat/segment_replication/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java index 3d6211ae50f1a..8f7f9e5bd20a7 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java @@ -69,7 +69,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/cluster_manager\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java index 781cfa425f369..9c054ffe1bcc7 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java @@ -71,7 +71,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/count\n"); sb.append("/_cat/count/{index}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java index cd54b169230d0..04bbdeeadc4c4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java @@ -82,7 +82,7 @@ public RestResponse buildResponse(NodesStatsResponse nodeStatses) throws Excepti } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/fielddata\n"); sb.append("/_cat/fielddata/{fields}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java index 898c3a05cec34..b4d336f4c10c0 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java @@ -69,7 +69,7 @@ public boolean allowSystemIndexAccessByDefault() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/health\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 6de67bb7d1add..8bf8e201cfc4d 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -61,8 +61,9 @@ import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; -import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; -import org.opensearch.rest.pagination.PaginatedQueryResponse; +import org.opensearch.rest.action.list.AbstractListAction; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageToken; import java.time.Instant; import java.time.ZoneOffset; @@ -89,7 +90,7 @@ * * @opensearch.api */ -public class RestIndicesAction extends AbstractCatAction { +public class RestIndicesAction extends AbstractListAction { private static final DateFormatter STRICT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_time"); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestIndicesAction.class); @@ -114,7 +115,7 @@ public boolean allowSystemIndexAccessByDefault() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/indices\n"); sb.append("/_cat/indices/{index}\n"); } @@ -177,14 +178,14 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { new ActionListener() { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { - IndexBasedPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); + IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); final String[] indicesToBeQueried = getIndicesToBeQueried(indices, paginationStrategy); final GroupedActionListener groupedListener = createGroupedListener( request, 4, listener, indicesToBeQueried, - getPaginatedQueryResponse(paginationStrategy) + getPageToken(paginationStrategy) ); groupedListener.onResponse(getSettingsResponse); groupedListener.onResponse(clusterStateResponse); @@ -309,7 +310,7 @@ private GroupedActionListener createGroupedListener( final int size, final ActionListener
listener, final String[] indicesToBeQueried, - final PaginatedQueryResponse paginatedQueryResponse + final PageToken pageToken ) { return new GroupedActionListener<>(new ActionListener>() { @Override @@ -340,7 +341,7 @@ public void onResponse(final Collection responses) { indicesStats, indicesStates, indicesToBeQueried, - paginatedQueryResponse + pageToken ); listener.onResponse(responseTable); } catch (Exception e) { @@ -373,8 +374,8 @@ protected Table getTableWithHeader(final RestRequest request) { return getTableWithHeader(request, null); } - protected Table getTableWithHeader(final RestRequest request, final PaginatedQueryResponse paginatedQueryResponse) { - Table table = new Table(paginatedQueryResponse); + protected Table getTableWithHeader(final RestRequest request, final PageToken pageToken) { + Table table = new Table(pageToken); table.startHeaders(); table.addCell("health", "alias:h;desc:current health status"); table.addCell("status", "alias:s;desc:open/close status"); @@ -745,10 +746,10 @@ protected Table buildTable( final Map indicesStats, final Map indicesMetadatas, final String[] indicesToBeQueried, - final PaginatedQueryResponse paginatedQueryResponse + final PageToken pageToken ) { final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request, paginatedQueryResponse); + final Table table = getTableWithHeader(request, pageToken); indicesSettings.forEach((indexName, settings) -> { if (indicesMetadatas.containsKey(indexName) == false) { @@ -1039,15 +1040,15 @@ private static A extractResponse(final Collection displayHeaders = buildDisplayHeaders(table, request); - if (Objects.nonNull(table.getPaginatedQueryResponse())) { - assert Objects.nonNull(table.getPaginatedQueryResponse().getPaginatedElement()) + if (Objects.nonNull(table.getPageToken())) { + assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) : "Paginated element is required in-case of paginated responses"; builder.startObject(); - builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPaginatedQueryResponse().getNextToken()); - builder.startArray(table.getPaginatedQueryResponse().getPaginatedElement()); + builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken()); + builder.startArray(table.getPageToken().getPaginatedEntity()); } else { builder.startArray(); } @@ -109,7 +109,7 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel builder.endObject(); } builder.endArray(); - if (Objects.nonNull(table.getPaginatedQueryResponse())) { + if (Objects.nonNull(table.getPageToken())) { builder.endObject(); } return new BytesRestResponse(RestStatus.OK, builder); @@ -151,8 +151,8 @@ public static RestResponse buildTextPlainResponse(Table table, RestChannel chann out.append("\n"); } // Adding a new row for next_token, in the response if the table is paginated. - if (Objects.nonNull(table.getPaginatedQueryResponse())) { - out.append("next_token" + " " + table.getPaginatedQueryResponse().getNextToken()); + if (Objects.nonNull(table.getPageToken())) { + out.append("next_token" + " " + table.getPageToken().getNextToken()); out.append("\n"); } out.close(); diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java index 2267f9200aece..0393dd15c8238 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestThreadPoolAction.java @@ -87,7 +87,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/thread_pool\n"); sb.append("/_cat/thread_pool/{thread_pools}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java new file mode 100644 index 0000000000000..1f5be657f425d --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.list; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.cat.AbstractCatAction; +import org.opensearch.rest.pagination.PageParams; + +import java.io.IOException; +import java.util.Objects; + +/** + * Base Transport action class for _list API. + * + * @opensearch.api + */ +public abstract class AbstractListAction extends AbstractCatAction { + + protected PageParams pageParams; + + protected abstract void documentation(StringBuilder sb); + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + boolean helpWanted = request.paramAsBoolean("help", false); + if (helpWanted || isActionPaginated() == false) { + return super.prepareRequest(request, client); + } + this.pageParams = validateAndGetPageParams(request); + assert Objects.nonNull(pageParams) : "pageParams can not be null for paginated queries"; + return doCatRequest(request, client); + } + + /** + * + * @return boolean denoting whether the RestAction will output paginated responses or not. + * Is kept false by default, every paginated action to override and return true. + */ + public boolean isActionPaginated() { + return false; + } + + /** + * + * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide + * its own implementation. Query params supported by the action specific to pagination along with the respective validations, + * should be added here. + */ + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + return null; + } + +} diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index a6c6775b0f47a..e70fdd77d8894 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -16,9 +16,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; -import org.opensearch.rest.pagination.IndexBasedPaginationStrategy; -import org.opensearch.rest.pagination.PaginatedQueryRequest; -import org.opensearch.rest.pagination.PaginatedQueryResponse; +import org.opensearch.rest.pagination.IndexPaginationStrategy; +import org.opensearch.rest.pagination.PageParams; +import org.opensearch.rest.pagination.PageToken; import java.util.List; import java.util.Map; @@ -27,6 +27,8 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; /** * _list API action to output indices in pages. @@ -35,8 +37,8 @@ */ public class RestIndicesListAction extends RestIndicesAction { - protected static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING = 1000; - protected static final int DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = 1000; + protected static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING = 5000; + protected static final int DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = 500; @Override public List routes() { @@ -60,28 +62,24 @@ public boolean isActionPaginated() { } @Override - protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest restRequest) { - PaginatedQueryRequest paginatedQueryRequest = restRequest.parsePaginatedQueryParams( - "ascending", - DEFAULT_LIST_INDICES_PAGE_SIZE_STRING - ); + protected PageParams validateAndGetPageParams(RestRequest restRequest) { + PageParams pageParams = restRequest.parsePaginatedQueryParams(PARAM_ASC_SORT_VALUE, DEFAULT_LIST_INDICES_PAGE_SIZE_STRING); // validating pageSize - if (paginatedQueryRequest.getSize() <= 0) { + if (pageParams.getSize() <= 0) { throw new IllegalArgumentException("size must be greater than zero"); - } else if (paginatedQueryRequest.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { + } else if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING + "]"); } // Validating sort order - if (!Objects.equals(paginatedQueryRequest.getSort(), "ascending") - && !Objects.equals(paginatedQueryRequest.getSort(), "descending")) { - throw new IllegalArgumentException("value of sort can either be ascending or descending"); + if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) { + throw new IllegalArgumentException("value of sort can either be asc or desc"); } // Next Token in the request will be validated by the IndexStrategyTokenParser itself. - if (Objects.nonNull(paginatedQueryRequest.getRequestedTokenStr())) { - IndexBasedPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(paginatedQueryRequest.getRequestedTokenStr()); + if (Objects.nonNull(pageParams.getRequestedToken())) { + IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken()); } - return paginatedQueryRequest; + return pageParams; } @Override @@ -92,7 +90,7 @@ protected Table buildTable( final Map indicesStats, final Map indicesMetadatas, final String[] indicesToBeQueried, - final PaginatedQueryResponse paginatedQueryResponse + final PageToken paginatedQueryResponse ) { final String healthParam = request.param("health"); final Table table = getTableWithHeader(request, paginatedQueryResponse); @@ -106,16 +104,16 @@ protected Table buildTable( } @Override - protected IndexBasedPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { - return new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterStateResponse.getState()); + protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clusterStateResponse) { + return new IndexPaginationStrategy(pageParams, clusterStateResponse.getState()); } @Override - protected PaginatedQueryResponse getPaginatedQueryResponse(IndexBasedPaginationStrategy paginationStrategy) { - return paginationStrategy.getPaginatedQueryResponse(); + protected PageToken getPageToken(IndexPaginationStrategy paginationStrategy) { + return paginationStrategy.getResponseToken(); } - protected String[] getIndicesToBeQueried(String[] indices, IndexBasedPaginationStrategy paginationStrategy) { - return paginationStrategy.getElementsFromRequestedToken().toArray(new String[0]); + protected String[] getIndicesToBeQueried(String[] indices, IndexPaginationStrategy paginationStrategy) { + return paginationStrategy.getRequestedEntities().toArray(new String[0]); } } diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java index 32e5824b45cec..4b8551ea7e14a 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestListAction.java @@ -13,7 +13,6 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.cat.AbstractCatAction; import java.io.IOException; import java.util.List; @@ -32,10 +31,10 @@ public class RestListAction extends BaseRestHandler { private static final String LIST_NL = LIST + "\n"; private final String HELP; - public RestListAction(List listActions) { + public RestListAction(List listActions) { StringBuilder sb = new StringBuilder(); sb.append(LIST_NL); - for (AbstractCatAction listAction : listActions) { + for (AbstractListAction listAction : listActions) { listAction.documentation(sb); } HELP = sb.toString(); diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java deleted file mode 100644 index 02c08a7a46a3b..0000000000000 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategy.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.pagination; - -import org.opensearch.OpenSearchParseException; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.Nullable; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_ASCENDING_SORT; - -/** - * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. - * The strategy considers create timestamps of indices as the keys to iterate over pages. - * - * @opensearch.internal - */ -public class IndexBasedPaginationStrategy implements PaginationStrategy { - - private final PaginatedQueryResponse paginatedQueryResponse; - private final List indicesFromRequestedToken; - - private static final String DEFAULT_INDICES_PAGINATED_ELEMENT = "indices"; - - public IndexBasedPaginationStrategy(PaginatedQueryRequest paginatedQueryRequest, ClusterState clusterState) { - // Get list of indices metadata sorted by their creation time and filtered by the last send index - List sortedIndicesList = PaginationStrategy.getSortedIndexMetadata( - clusterState, - getMetadataListFilter(paginatedQueryRequest.getRequestedTokenStr(), paginatedQueryRequest.getSort()), - getMetadataListComparator(paginatedQueryRequest.getSort()) - ); - List metadataListForRequestedToken = getMetadataListForRequestedToken(sortedIndicesList, paginatedQueryRequest); - this.indicesFromRequestedToken = metadataListForRequestedToken.stream() - .map(metadata -> metadata.getIndex().getName()) - .collect(Collectors.toList()); - this.paginatedQueryResponse = getPaginatedResponseForRequestedToken(paginatedQueryRequest.getSize(), sortedIndicesList); - } - - private static Predicate getMetadataListFilter(String requestedTokenStr, String sortOrder) { - boolean isAscendingSort = sortOrder.equals(PAGINATED_QUERY_ASCENDING_SORT); - IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() - ? null - : new IndexStrategyToken(requestedTokenStr); - if (Objects.isNull(requestedToken)) { - return indexMetadata -> true; - } - return indexMetadata -> { - if (indexMetadata.getIndex().getName().equals(requestedToken.nameOfLastRespondedIndex)) { - return false; - } else if (indexMetadata.getCreationDate() == requestedToken.creationTimeOfLastRespondedIndex) { - return isAscendingSort - ? indexMetadata.getIndex().getName().compareTo(requestedToken.nameOfLastRespondedIndex) > 0 - : indexMetadata.getIndex().getName().compareTo(requestedToken.nameOfLastRespondedIndex) < 0; - } - return isAscendingSort - ? indexMetadata.getCreationDate() > requestedToken.creationTimeOfLastRespondedIndex - : indexMetadata.getCreationDate() < requestedToken.creationTimeOfLastRespondedIndex; - }; - } - - private static Comparator getMetadataListComparator(String sortOrder) { - boolean isAscendingSort = sortOrder.equals(PAGINATED_QUERY_ASCENDING_SORT); - return (metadata1, metadata2) -> { - if (metadata1.getCreationDate() == metadata2.getCreationDate()) { - return isAscendingSort - ? metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()) - : metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); - } - return isAscendingSort - ? Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()) - : Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()); - }; - } - - private List getMetadataListForRequestedToken( - List sortedIndicesList, - PaginatedQueryRequest paginatedQueryRequest - ) { - if (sortedIndicesList.isEmpty()) { - return new ArrayList<>(); - } - return sortedIndicesList.subList(0, Math.min(paginatedQueryRequest.getSize(), sortedIndicesList.size())); - } - - private PaginatedQueryResponse getPaginatedResponseForRequestedToken(int pageSize, List sortedIndicesList) { - if (sortedIndicesList.size() <= pageSize) { - return new PaginatedQueryResponse(null, DEFAULT_INDICES_PAGINATED_ELEMENT); - } - return new PaginatedQueryResponse( - new IndexStrategyToken( - sortedIndicesList.get(pageSize - 1).getCreationDate(), - sortedIndicesList.get(pageSize - 1).getIndex().getName() - ).generateEncryptedToken(), - DEFAULT_INDICES_PAGINATED_ELEMENT - ); - } - - @Override - @Nullable - public PaginatedQueryResponse getPaginatedQueryResponse() { - return paginatedQueryResponse; - } - - @Override - public List getElementsFromRequestedToken() { - return Objects.isNull(indicesFromRequestedToken) ? new ArrayList<>() : indicesFromRequestedToken; - } - - /** - * TokenParser to be used by {@link IndexBasedPaginationStrategy}. - * Token would like: IndexNumberToStartTheNextPageFrom + | + CreationTimeOfLastRespondedIndex + | + - * QueryStartTime + | + NameOfLastRespondedIndex - */ - public static class IndexStrategyToken { - - private static final String TOKEN_JOIN_DELIMITER = "|"; - private static final String TOKEN_SPLIT_REGEX = "\\|"; - private static final int CREATION_TIME_FIELD_POSITION_IN_TOKEN = 0; - private static final int INDEX_NAME_FIELD_POSITION_IN_TOKEN = 1; - - /** - * Represents creation times of last index which was displayed in the previous page. - * Used to identify the new start point in case the indices get created/deleted while queries are executed. - */ - private final long creationTimeOfLastRespondedIndex; - - /** - * Represents name of the last index which was displayed in the previous page. - * Used to identify whether the sorted list of indices has changed or not. - */ - private final String nameOfLastRespondedIndex; - - public IndexStrategyToken(String requestedTokenString) { - validateIndexStrategyToken(requestedTokenString); - String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); - final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); - this.creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATION_TIME_FIELD_POSITION_IN_TOKEN]); - this.nameOfLastRespondedIndex = decryptedTokenElements[INDEX_NAME_FIELD_POSITION_IN_TOKEN]; - } - - public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { - Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); - this.creationTimeOfLastRespondedIndex = creationTimeOfLastRespondedIndex; - this.nameOfLastRespondedIndex = nameOfLastRespondedIndex; - } - - public String generateEncryptedToken() { - return PaginationStrategy.encryptStringToken( - String.join(TOKEN_JOIN_DELIMITER, String.valueOf(creationTimeOfLastRespondedIndex), nameOfLastRespondedIndex) - ); - } - - /** - * Will perform simple validations on token received in the request and initialize the data members. - * The token should be base64 encoded, and should contain the expected number of elements separated by "$". - * The timestamps should also be a valid long. - * - * @param requestedTokenString string denoting the encoded next token requested by the user - */ - public static void validateIndexStrategyToken(String requestedTokenString) { - Objects.requireNonNull(requestedTokenString, "requestedTokenString can not be null"); - String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); - final String[] decryptedTokenElements = decryptedToken.split(TOKEN_SPLIT_REGEX); - if (decryptedTokenElements.length != 2) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } - try { - long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATION_TIME_FIELD_POSITION_IN_TOKEN]); - if (creationTimeOfLastRespondedIndex <= 0) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } - } catch (NumberFormatException exception) { - throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); - } - } - } - -} diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java new file mode 100644 index 0000000000000..3a84226dc85cf --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.pagination; + +import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Nullable; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; + +/** + * This strategy can be used by the Rest APIs wanting to paginate the responses based on Indices. + * The strategy considers create timestamps of indices as the keys to iterate over pages. + * + * @opensearch.internal + */ +public class IndexPaginationStrategy implements PaginationStrategy { + private static final String DEFAULT_INDICES_PAGINATED_ELEMENT = "indices"; + + private static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata1.getIndex().getName().compareTo(metadata2.getIndex().getName()); + } + return Long.compare(metadata1.getCreationDate(), metadata2.getCreationDate()); + }; + private static final Comparator DESC_COMPARATOR = (metadata1, metadata2) -> { + if (metadata1.getCreationDate() == metadata2.getCreationDate()) { + return metadata2.getIndex().getName().compareTo(metadata1.getIndex().getName()); + } + return Long.compare(metadata2.getCreationDate(), metadata1.getCreationDate()); + }; + + private final PageToken pageToken; + private final List requestedIndices; + + public IndexPaginationStrategy(PageParams pageParams, ClusterState clusterState) { + // Get list of indices metadata sorted by their creation time and filtered by the last sent index + List sortedIndices = PaginationStrategy.getSortedIndexMetadata( + clusterState, + getMetadataFilter(pageParams.getRequestedToken(), pageParams.getSort()), + PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) ? ASC_COMPARATOR : DESC_COMPARATOR + ); + // Trim sortedIndicesList to get the list of indices metadata to be sent as response + List metadataSublist = getMetadataSubList(sortedIndices, pageParams.getSize()); + // Get list of index names from the trimmed metadataSublist + this.requestedIndices = metadataSublist.stream().map(metadata -> metadata.getIndex().getName()).collect(Collectors.toList()); + this.pageToken = getResponseToken( + pageParams.getSize(), + sortedIndices.size(), + metadataSublist.isEmpty() ? null : metadataSublist.get(metadataSublist.size() - 1) + ); + } + + private static Predicate getMetadataFilter(String requestedTokenStr, String sortOrder) { + boolean isAscendingSort = sortOrder.equals(PARAM_ASC_SORT_VALUE); + IndexStrategyToken requestedToken = Objects.isNull(requestedTokenStr) || requestedTokenStr.isEmpty() + ? null + : new IndexStrategyToken(requestedTokenStr); + if (Objects.isNull(requestedToken)) { + return indexMetadata -> true; + } + return metadata -> { + if (metadata.getIndex().getName().equals(requestedToken.lastIndexName)) { + return false; + } else if (metadata.getCreationDate() == requestedToken.lastIndexCreationTime) { + return isAscendingSort + ? metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) > 0 + : metadata.getIndex().getName().compareTo(requestedToken.lastIndexName) < 0; + } + return isAscendingSort + ? metadata.getCreationDate() > requestedToken.lastIndexCreationTime + : metadata.getCreationDate() < requestedToken.lastIndexCreationTime; + }; + } + + private List getMetadataSubList(List sortedIndices, final int pageSize) { + if (sortedIndices.isEmpty()) { + return new ArrayList<>(); + } + return sortedIndices.subList(0, Math.min(pageSize, sortedIndices.size())); + } + + private PageToken getResponseToken(final int pageSize, final int totalIndices, IndexMetadata lastIndex) { + if (totalIndices <= pageSize) { + return new PageToken(null, DEFAULT_INDICES_PAGINATED_ELEMENT); + } + return new PageToken( + new IndexStrategyToken(lastIndex.getCreationDate(), lastIndex.getIndex().getName()).generateEncryptedToken(), + DEFAULT_INDICES_PAGINATED_ELEMENT + ); + } + + @Override + @Nullable + public PageToken getResponseToken() { + return pageToken; + } + + @Override + public List getRequestedEntities() { + return Objects.isNull(requestedIndices) ? new ArrayList<>() : requestedIndices; + } + + /** + * TokenParser to be used by {@link IndexPaginationStrategy}. + * Token would look like: IndexNumberToStartTheNextPageFrom + | + CreationTimeOfLastRespondedIndex + | + + * QueryStartTime + | + NameOfLastRespondedIndex + */ + public static class IndexStrategyToken { + + private static final String JOIN_DELIMITER = "|"; + private static final String SPLIT_REGEX = "\\|"; + private static final int CREATE_TIME_POS_IN_TOKEN = 0; + private static final int INDEX_NAME_POS_IN_TOKEN = 1; + + /** + * Represents creation times of last index which was displayed in the page. + * Used to identify the new start point in case the indices get created/deleted while queries are executed. + */ + private final long lastIndexCreationTime; + + /** + * Represents name of the last index which was displayed in the page. + * Used to identify whether the sorted list of indices has changed or not. + */ + private final String lastIndexName; + + public IndexStrategyToken(String requestedTokenString) { + validateIndexStrategyToken(requestedTokenString); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + this.lastIndexCreationTime = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + this.lastIndexName = decryptedTokenElements[INDEX_NAME_POS_IN_TOKEN]; + } + + public IndexStrategyToken(long creationTimeOfLastRespondedIndex, String nameOfLastRespondedIndex) { + Objects.requireNonNull(nameOfLastRespondedIndex, "index name should be provided"); + this.lastIndexCreationTime = creationTimeOfLastRespondedIndex; + this.lastIndexName = nameOfLastRespondedIndex; + } + + public String generateEncryptedToken() { + return PaginationStrategy.encryptStringToken(String.join(JOIN_DELIMITER, String.valueOf(lastIndexCreationTime), lastIndexName)); + } + + /** + * Will perform simple validations on token received in the request. + * Token should be base64 encoded, and should contain the expected number of elements separated by "|". + * Timestamps should also be a valid long. + * + * @param requestedTokenStr string denoting the encoded token requested by the user. + */ + public static void validateIndexStrategyToken(String requestedTokenStr) { + Objects.requireNonNull(requestedTokenStr, "requestedTokenString can not be null"); + String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenStr); + final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); + if (decryptedTokenElements.length != 2) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + try { + long creationTimeOfLastRespondedIndex = Long.parseLong(decryptedTokenElements[CREATE_TIME_POS_IN_TOKEN]); + if (creationTimeOfLastRespondedIndex <= 0) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } catch (NumberFormatException exception) { + throw new OpenSearchParseException(INCORRECT_TAINTED_NEXT_TOKEN_ERROR_MESSAGE); + } + } + } + +} diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java similarity index 63% rename from server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java rename to server/src/main/java/org/opensearch/rest/pagination/PageParams.java index eb18429676c1c..9b2074bc3fed0 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryRequest.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PageParams.java @@ -15,18 +15,19 @@ * Class specific to paginated queries, which will contain common query params required by a paginated API. */ @PublicApi(since = "3.0.0") -public class PaginatedQueryRequest { +public class PageParams { - public static final String PAGINATED_QUERY_PARAM_SORT_KEY = "sort"; - public static final String PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY = "next_token"; - public static final String PAGINATED_QUERY_PARAM_SIZE_KEY = "size"; - public static final String PAGINATED_QUERY_ASCENDING_SORT = "ascending"; + public static final String PARAM_SORT = "sort"; + public static final String PARAM_NEXT_TOKEN = "next_token"; + public static final String PARAM_SIZE = "size"; + public static final String PARAM_ASC_SORT_VALUE = "asc"; + public static final String PARAM_DESC_SORT_VALUE = "desc"; private final String requestedTokenStr; private final String sort; private final int size; - public PaginatedQueryRequest(String requestedToken, String sort, int size) { + public PageParams(String requestedToken, String sort, int size) { this.requestedTokenStr = requestedToken; this.sort = sort; this.size = size; @@ -36,7 +37,7 @@ public String getSort() { return sort; } - public String getRequestedTokenStr() { + public String getRequestedToken() { return requestedTokenStr; } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java similarity index 76% rename from server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java rename to server/src/main/java/org/opensearch/rest/pagination/PageToken.java index b9abe839b7230..d62e1be695715 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginatedQueryResponse.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PageToken.java @@ -12,7 +12,7 @@ * Pagination response metadata for a paginated query. * @opensearch.internal */ -public class PaginatedQueryResponse { +public class PageToken { public static final String PAGINATED_RESPONSE_NEXT_TOKEN_KEY = "next_token"; @@ -24,19 +24,19 @@ public class PaginatedQueryResponse { /** * String denoting the element which is being paginated (for e.g. shards, indices..). */ - private final String paginatedElement; + private final String paginatedEntity; - public PaginatedQueryResponse(String nextToken, String paginatedElement) { + public PageToken(String nextToken, String paginatedElement) { assert paginatedElement != null : "paginatedElement must be specified for a paginated response"; this.nextToken = nextToken; - this.paginatedElement = paginatedElement; + this.paginatedEntity = paginatedElement; } public String getNextToken() { return nextToken; } - public String getPaginatedElement() { - return paginatedElement; + public String getPaginatedEntity() { + return paginatedEntity; } } diff --git a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java index 8675b85236755..7f9825a7cc09b 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/PaginationStrategy.java @@ -35,13 +35,13 @@ public interface PaginationStrategy { * * @return Base64 encoded string, which can be used to fetch next page of response. */ - PaginatedQueryResponse getPaginatedQueryResponse(); + PageToken getResponseToken(); /** * * @return List of elements fetched corresponding to the store and token received by the strategy. */ - List getElementsFromRequestedToken(); + List getRequestedEntities(); /** * diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java similarity index 54% rename from server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java rename to server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java index 8def1420d86c4..575874cbe6a47 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexBasedPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java @@ -27,7 +27,7 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; -public class IndexBasedPaginationStrategyTests extends OpenSearchTestCase { +public class IndexPaginationStrategyTests extends OpenSearchTestCase { public void testRetrieveAllIndicesInAscendingOrder() { List indexNumberList = new ArrayList<>(); @@ -46,21 +46,21 @@ public void testRetrieveAllIndicesInAscendingOrder() { String requestedToken = null; int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(requestedToken, "ascending", pageSize); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); + PageParams pageParams = new PageParams(requestedToken, "asc", pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); if (pageNumber < totalPagesToFetch) { - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); } else { - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); } - requestedToken = paginationStrategy.getPaginatedQueryResponse().getNextToken(); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); // Asserting all the indices received int responseItr = 0; for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(100, pageNumber * pageSize); indexNumber++) { - assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getElementsFromRequestedToken().get(responseItr)); + assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); responseItr++; } - assertEquals(responseItr, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); } } } @@ -83,22 +83,22 @@ public void testRetrieveAllIndicesInDescendingOrder() { int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); int startIndexNumber = totalIndices; for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(requestedToken, "descending", pageSize); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); + PageParams pageParams = new PageParams(requestedToken, "desc", pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); if (pageNumber < totalPagesToFetch) { - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); } else { - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); } - requestedToken = paginationStrategy.getPaginatedQueryResponse().getNextToken(); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); // Asserting all the indices received int responseItr = 0; int endIndexNumberForPage = Math.max(startIndexNumber - pageSize, 0); for (; startIndexNumber > endIndexNumberForPage; startIndexNumber--) { - assertEquals("test-index-" + startIndexNumber, paginationStrategy.getElementsFromRequestedToken().get(responseItr)); + assertEquals("test-index-" + startIndexNumber, paginationStrategy.getRequestedEntities().get(responseItr)); responseItr++; } - assertEquals(responseItr, paginationStrategy.getElementsFromRequestedToken().size()); + assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); } } } @@ -106,108 +106,108 @@ public void testRetrieveAllIndicesInDescendingOrder() { public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { // Query1 with 4 indices in clusterState (test-index1,2,3,4) ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + PageParams pageParams = new PageParams(null, "asc", 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Adding index5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-2 which has already been displayed, still test-index-2 should get displayed clusterState = deleteIndexFromClusterState(clusterState, 2); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query // instead test-index-5 should now get displayed. clusterState = deleteIndexFromClusterState(clusterState, 4); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + assertNull(paginationStrategy.getResponseToken().getNextToken()); } public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDescOrder() { // Query1 with 4 indices in clusterState (test-index1,2,3,4). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "descending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + PageParams pageParams = new PageParams(null, "desc", 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // adding test-index-5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-3", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-3 which has already been displayed, still index2 should get displayed. clusterState = deleteIndexFromClusterState(clusterState, 3); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. clusterState = deleteIndexFromClusterState(clusterState, 1); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "descending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(0, paginationStrategy.getElementsFromRequestedToken().size()); - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(0, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); } public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() { // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); - PaginatedQueryRequest paginatedQueryRequest = new PaginatedQueryRequest(null, "ascending", 1); - IndexBasedPaginationStrategy paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-1", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + PageParams pageParams = new PageParams(null, "asc", 1); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // executing next query without any changes to clusterState - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-2", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-1, test-index-2 & test-index-3 and executing next query. test-index-4 should get displayed. clusterState = deleteIndexFromClusterState(clusterState, 1); clusterState = deleteIndexFromClusterState(clusterState, 2); clusterState = deleteIndexFromClusterState(clusterState, 3); - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-4", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNotNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. - paginatedQueryRequest = new PaginatedQueryRequest(paginationStrategy.getPaginatedQueryResponse().getNextToken(), "ascending", 1); - paginationStrategy = new IndexBasedPaginationStrategy(paginatedQueryRequest, clusterState); - assertEquals(1, paginationStrategy.getElementsFromRequestedToken().size()); - assertEquals("test-index-5", paginationStrategy.getElementsFromRequestedToken().get(0)); - assertNull(paginationStrategy.getPaginatedQueryResponse().getNextToken()); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); + assertNull(paginationStrategy.getResponseToken().getNextToken()); } public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { try { - new IndexBasedPaginationStrategy.IndexStrategyToken(null); + new IndexPaginationStrategy.IndexStrategyToken(null); fail("expected exception"); } catch (Exception e) { assert e.getMessage().contains("requestedTokenString can not be null"); @@ -215,30 +215,30 @@ public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { } public void testIndexStrategyPageTokenWithWronglyEncryptedRequestToken() { - assertThrows(OpenSearchParseException.class, () -> new IndexBasedPaginationStrategy.IndexStrategyToken("3%4%5")); + assertThrows(OpenSearchParseException.class, () -> new IndexPaginationStrategy.IndexStrategyToken("3%4%5")); } public void testIndexStrategyPageTokenWithIncorrectNumberOfElementsInRequestedToken() { assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543")) + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1725361543")) ); assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("1|1725361543|index|12345")) ); } public void testIndexStrategyPageTokenWithInvalidValuesInRequestedToken() { assertThrows( OpenSearchParseException.class, - () -> new IndexBasedPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index")) + () -> new IndexPaginationStrategy.IndexStrategyToken(PaginationStrategy.encryptStringToken("-1725361543|index")) ); } public void testCreatingIndexStrategyPageTokenWithNameOfLastRespondedIndexNull() { try { - new IndexBasedPaginationStrategy.IndexStrategyToken(1234l, null); + new IndexPaginationStrategy.IndexStrategyToken(1234l, null); fail("expected exception"); } catch (Exception e) { assert e.getMessage().contains("index name should be provided"); From 183850685957da9407ee41d1e0e8917c8cc0e75e Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Tue, 24 Sep 2024 10:13:40 +0530 Subject: [PATCH 08/12] Refactoring RestIndicesAction Signed-off-by: Harsh Garg --- .../java/org/opensearch/rest/RestHandler.java | 7 + .../rest/action/cat/RestIndicesAction.java | 434 +++++++++--------- .../opensearch/rest/action/cat/RestTable.java | 38 +- .../rest/action/list/AbstractListAction.java | 36 +- .../action/list/RestIndicesListAction.java | 67 ++- .../action/cat/RestIndicesActionTests.java | 63 ++- .../rest/action/cat/RestTableTests.java | 93 +++- .../IndexPaginationStrategyTests.java | 262 ++++++++--- 8 files changed, 632 insertions(+), 368 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java index 1139e5fc65f31..143cbd472ed07 100644 --- a/server/src/main/java/org/opensearch/rest/RestHandler.java +++ b/server/src/main/java/org/opensearch/rest/RestHandler.java @@ -125,6 +125,13 @@ default boolean allowSystemIndexAccessByDefault() { return false; } + /** + * Denotes whether the RestHandler will output paginated responses or not. + */ + default boolean isActionPaginated() { + return false; + } + static RestHandler wrapper(RestHandler delegate) { return new Wrapper(delegate); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java index 8bf8e201cfc4d..1e76008ff8c64 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestIndicesAction.java @@ -50,6 +50,7 @@ import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Table; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; @@ -71,9 +72,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.Spliterators; import java.util.function.Function; @@ -179,13 +182,18 @@ public void onResponse(final GetSettingsResponse getSettingsResponse) { @Override public void onResponse(ClusterStateResponse clusterStateResponse) { IndexPaginationStrategy paginationStrategy = getPaginationStrategy(clusterStateResponse); - final String[] indicesToBeQueried = getIndicesToBeQueried(indices, paginationStrategy); + // For non-paginated queries, indicesToBeQueried would be same as indices retrieved from + // rest request and unresolved, while for paginated queries, it would be a list of indices + // already resolved by ClusterStateRequest and to be displayed in a page. + final String[] indicesToBeQueried = Objects.isNull(paginationStrategy) + ? indices + : paginationStrategy.getRequestedEntities().toArray(new String[0]); final GroupedActionListener groupedListener = createGroupedListener( request, 4, listener, indicesToBeQueried, - getPageToken(paginationStrategy) + Objects.isNull(paginationStrategy) ? null : paginationStrategy.getResponseToken() ); groupedListener.onResponse(getSettingsResponse); groupedListener.onResponse(clusterStateResponse); @@ -340,7 +348,7 @@ public void onResponse(final Collection responses) { indicesHealths, indicesStats, indicesStates, - indicesToBeQueried, + getTableIterator(indicesToBeQueried, indicesSettings), pageToken ); listener.onResponse(responseTable); @@ -745,294 +753,287 @@ protected Table buildTable( final Map indicesHealths, final Map indicesStats, final Map indicesMetadatas, - final String[] indicesToBeQueried, + final Iterator> tableIterator, final PageToken pageToken ) { final String healthParam = request.param("health"); final Table table = getTableWithHeader(request, pageToken); - indicesSettings.forEach((indexName, settings) -> { + while (tableIterator.hasNext()) { + final Tuple tuple = tableIterator.next(); + String indexName = tuple.v1(); + Settings settings = tuple.v2(); + if (indicesMetadatas.containsKey(indexName) == false) { // the index exists in the Get Indices response but is not present in the cluster state: // it is likely that the index was deleted in the meanwhile, so we ignore it. - return; + continue; } - buildRow(indicesSettings, indicesHealths, indicesStats, indicesMetadatas, healthParam, indexName, table); - }); - - return table; - } - protected void buildRow( - final Map indicesSettings, - final Map indicesHealths, - final Map indicesStats, - final Map indicesMetadatas, - final String healthParam, - final String indexName, - Table table - ) { - Settings settings = indicesSettings.get(indexName); - final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); - final IndexMetadata.State indexState = indexMetadata.getState(); - final IndexStats indexStats = indicesStats.get(indexName); - final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); - - final String health; - final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); - if (indexHealth != null) { - health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); - } else if (indexStats != null) { - health = "red*"; - } else { - health = ""; - } + final IndexMetadata indexMetadata = indicesMetadatas.get(indexName); + final IndexMetadata.State indexState = indexMetadata.getState(); + final IndexStats indexStats = indicesStats.get(indexName); + final boolean searchThrottled = IndexSettings.INDEX_SEARCH_THROTTLED.get(settings); - if (healthParam != null) { - final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); - boolean skip; + final String health; + final ClusterIndexHealth indexHealth = indicesHealths.get(indexName); if (indexHealth != null) { - // index health is known but does not match the one requested - skip = indexHealth.getStatus() != healthStatusFilter; + health = indexHealth.getStatus().toString().toLowerCase(Locale.ROOT); + } else if (indexStats != null) { + health = "red*"; } else { - // index health is unknown, skip if we don't explicitly request RED health - skip = ClusterHealthStatus.RED != healthStatusFilter; + health = ""; } - if (skip) { - return; + + if (healthParam != null) { + final ClusterHealthStatus healthStatusFilter = ClusterHealthStatus.fromString(healthParam); + boolean skip; + if (indexHealth != null) { + // index health is known but does not match the one requested + skip = indexHealth.getStatus() != healthStatusFilter; + } else { + // index health is unknown, skip if we don't explicitly request RED health + skip = ClusterHealthStatus.RED != healthStatusFilter; + } + if (skip) { + continue; + } } - } - final CommonStats primaryStats; - final CommonStats totalStats; + final CommonStats primaryStats; + final CommonStats totalStats; - if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { - // TODO: expose docs stats for replicated closed indices - primaryStats = new CommonStats(); - totalStats = new CommonStats(); - } else { - primaryStats = indexStats.getPrimaries(); - totalStats = indexStats.getTotal(); - } - table.startRow(); - table.addCell(health); - table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); - table.addCell(indexName); - table.addCell(indexMetadata.getIndexUUID()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); - table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); + if (indexStats == null || indexState == IndexMetadata.State.CLOSE) { + // TODO: expose docs stats for replicated closed indices + primaryStats = new CommonStats(); + totalStats = new CommonStats(); + } else { + primaryStats = indexStats.getPrimaries(); + totalStats = indexStats.getTotal(); + } + table.startRow(); + table.addCell(health); + table.addCell(indexState.toString().toLowerCase(Locale.ROOT)); + table.addCell(indexName); + table.addCell(indexMetadata.getIndexUUID()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards()); + table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); - table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getCount()); + table.addCell(primaryStats.getDocs() == null ? null : primaryStats.getDocs().getDeleted()); - table.addCell(indexMetadata.getCreationDate()); - ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); - table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); + table.addCell(indexMetadata.getCreationDate()); + ZonedDateTime creationTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(indexMetadata.getCreationDate()), ZoneOffset.UTC); + table.addCell(STRICT_DATE_TIME_FORMATTER.format(creationTime)); - table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); - table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); + table.addCell(totalStats.getStore() == null ? null : totalStats.getStore().size()); + table.addCell(primaryStats.getStore() == null ? null : primaryStats.getStore().size()); - table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); - table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); + table.addCell(totalStats.getCompletion() == null ? null : totalStats.getCompletion().getSize()); + table.addCell(primaryStats.getCompletion() == null ? null : primaryStats.getCompletion().getSize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getMemorySize()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getMemorySize()); - table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); - table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); + table.addCell(totalStats.getFieldData() == null ? null : totalStats.getFieldData().getEvictions()); + table.addCell(primaryStats.getFieldData() == null ? null : primaryStats.getFieldData().getEvictions()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getMemorySize()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getMemorySize()); - table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); - table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); + table.addCell(totalStats.getQueryCache() == null ? null : totalStats.getQueryCache().getEvictions()); + table.addCell(primaryStats.getQueryCache() == null ? null : primaryStats.getQueryCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMemorySize()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMemorySize()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getEvictions()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getEvictions()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getHitCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getHitCount()); - table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); - table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); + table.addCell(totalStats.getRequestCache() == null ? null : totalStats.getRequestCache().getMissCount()); + table.addCell(primaryStats.getRequestCache() == null ? null : primaryStats.getRequestCache().getMissCount()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotal()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotal()); - table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); - table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); + table.addCell(totalStats.getFlush() == null ? null : totalStats.getFlush().getTotalTime()); + table.addCell(primaryStats.getFlush() == null ? null : primaryStats.getFlush().getTotalTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().current()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().current()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getExistsCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getExistsCount()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingTime()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingTime()); - table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); - table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); + table.addCell(totalStats.getGet() == null ? null : totalStats.getGet().getMissingCount()); + table.addCell(primaryStats.getGet() == null ? null : primaryStats.getGet().getMissingCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getDeleteCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getDeleteCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCurrent()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCurrent()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexTime()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexTime()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexCount()); - table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(totalStats.getIndexing() == null ? null : totalStats.getIndexing().getTotal().getIndexFailedCount()); + table.addCell(primaryStats.getIndexing() == null ? null : primaryStats.getIndexing().getTotal().getIndexFailedCount()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrent()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrent()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getCurrentSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getCurrentSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotal()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotal()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalNumDocs()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalNumDocs()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalSize()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalSize()); - table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); - table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); + table.addCell(totalStats.getMerge() == null ? null : totalStats.getMerge().getTotalTime()); + table.addCell(primaryStats.getMerge() == null ? null : primaryStats.getMerge().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotal()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotal()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getExternalTotalTime()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getExternalTotalTime()); - table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); - table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); + table.addCell(totalStats.getRefresh() == null ? null : totalStats.getRefresh().getListeners()); + table.addCell(primaryStats.getRefresh() == null ? null : primaryStats.getRefresh().getListeners()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getFetchCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getFetchCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getOpenContexts()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getOpenContexts()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentQueryCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentQueryCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getConcurrentAvgSliceCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getConcurrentAvgSliceCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getScrollCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getScrollCount()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getPitCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getPitCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getCount()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getCount()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getZeroMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getZeroMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getIndexWriterMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getIndexWriterMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getVersionMapMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getVersionMapMemory()); - table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); - table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); + table.addCell(totalStats.getSegments() == null ? null : totalStats.getSegments().getBitsetMemory()); + table.addCell(primaryStats.getSegments() == null ? null : primaryStats.getSegments().getBitsetMemory()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().current()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().current()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().total()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().total()); - table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); - table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); + table.addCell(totalStats.getWarmer() == null ? null : totalStats.getWarmer().totalTime()); + table.addCell(primaryStats.getWarmer() == null ? null : primaryStats.getWarmer().totalTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCurrent()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCurrent()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestTime()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestTime()); - table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); - table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); + table.addCell(totalStats.getSearch() == null ? null : totalStats.getSearch().getTotal().getSuggestCount()); + table.addCell(primaryStats.getSearch() == null ? null : primaryStats.getSearch().getTotal().getSuggestCount()); - table.addCell(totalStats.getTotalMemory()); - table.addCell(primaryStats.getTotalMemory()); + table.addCell(totalStats.getTotalMemory()); + table.addCell(primaryStats.getTotalMemory()); - table.addCell(searchThrottled); + table.addCell(searchThrottled); - table.endRow(); + table.endRow(); + + } + + return table; } @SuppressWarnings("unchecked") @@ -1040,16 +1041,33 @@ private static A extractResponse(final Collection> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + final Iterator settingsMapIter = indexSettingsMap.keySet().iterator(); + + @Override + public boolean hasNext() { + return settingsMapIter.hasNext(); + } + + @Override + public Tuple next() { + String index = settingsMapIter.next(); + return new Tuple<>(index, indexSettingsMap.get(index)); + } + }; } } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java index 6719b2c47b6c0..d622dd7a956f4 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTable.java @@ -90,16 +90,37 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel RestRequest request = channel.request(); XContentBuilder builder = channel.newBuilder(); List displayHeaders = buildDisplayHeaders(table, request); - if (Objects.nonNull(table.getPageToken())) { - assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) - : "Paginated element is required in-case of paginated responses"; - builder.startObject(); - builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken()); - builder.startArray(table.getPageToken().getPaginatedEntity()); + buildPaginatedXContentBuilder(table, request, builder, displayHeaders); } else { builder.startArray(); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); } + return new BytesRestResponse(RestStatus.OK, builder); + } + + private static void buildPaginatedXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { + assert Objects.nonNull(table.getPageToken().getPaginatedEntity()) : "Paginated element is required in-case of paginated responses"; + builder.startObject(); + builder.field(PAGINATED_RESPONSE_NEXT_TOKEN_KEY, table.getPageToken().getNextToken()); + builder.startArray(table.getPageToken().getPaginatedEntity()); + addRowsToXContentBuilder(table, request, builder, displayHeaders); + builder.endArray(); + builder.endObject(); + } + + private static void addRowsToXContentBuilder( + Table table, + RestRequest request, + XContentBuilder builder, + List displayHeaders + ) throws Exception { List rowOrder = getRowOrder(table, request); for (Integer row : rowOrder) { builder.startObject(); @@ -108,11 +129,6 @@ public static RestResponse buildXContentBuilder(Table table, RestChannel channel } builder.endObject(); } - builder.endArray(); - if (Objects.nonNull(table.getPageToken())) { - builder.endObject(); - } - return new BytesRestResponse(RestStatus.OK, builder); } public static RestResponse buildTextPlainResponse(Table table, RestChannel channel) throws IOException { diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java index 1f5be657f425d..6b34a7585c658 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -16,6 +16,9 @@ import java.io.IOException; import java.util.Objects; +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; + /** * Base Transport action class for _list API. * @@ -23,6 +26,7 @@ */ public abstract class AbstractListAction extends AbstractCatAction { + private static final int DEFAULT_PAGE_SIZE = 100; protected PageParams pageParams; protected abstract void documentation(StringBuilder sb); @@ -38,23 +42,35 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return doCatRequest(request, client); } - /** - * - * @return boolean denoting whether the RestAction will output paginated responses or not. - * Is kept false by default, every paginated action to override and return true. - */ + @Override public boolean isActionPaginated() { - return false; + return true; } /** * - * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide - * its own implementation. Query params supported by the action specific to pagination along with the respective validations, - * should be added here. + * @return Metadata that can be extracted out from the rest request. Query params supported by the action specific + * to pagination along with any respective validations to be added here. */ protected PageParams validateAndGetPageParams(RestRequest restRequest) { - return null; + PageParams pageParams = restRequest.parsePaginatedQueryParams(defaultSort(), defaultPageSize()); + // validating pageSize + if (pageParams.getSize() <= 0) { + throw new IllegalArgumentException("size must be greater than zero"); + } + // Validating sort order + if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) { + throw new IllegalArgumentException("value of sort can either be asc or desc"); + } + return pageParams; + } + + protected int defaultPageSize() { + return DEFAULT_PAGE_SIZE; + } + + protected String defaultSort() { + return PARAM_ASC_SORT_VALUE; } } diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index e70fdd77d8894..7964d5f3dcdee 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -9,17 +9,14 @@ package org.opensearch.rest.action.list; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.action.admin.indices.stats.IndexStats; -import org.opensearch.cluster.health.ClusterIndexHealth; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.Table; +import org.opensearch.common.collect.Tuple; import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.cat.RestIndicesAction; import org.opensearch.rest.pagination.IndexPaginationStrategy; import org.opensearch.rest.pagination.PageParams; -import org.opensearch.rest.pagination.PageToken; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -27,8 +24,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static org.opensearch.rest.RestRequest.Method.GET; -import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; -import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; /** * _list API action to output indices in pages. @@ -63,44 +58,20 @@ public boolean isActionPaginated() { @Override protected PageParams validateAndGetPageParams(RestRequest restRequest) { - PageParams pageParams = restRequest.parsePaginatedQueryParams(PARAM_ASC_SORT_VALUE, DEFAULT_LIST_INDICES_PAGE_SIZE_STRING); - // validating pageSize - if (pageParams.getSize() <= 0) { - throw new IllegalArgumentException("size must be greater than zero"); - } else if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { + PageParams pageParams = super.validateAndGetPageParams(restRequest); + // validate max supported pageSize + if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING + "]"); } - // Validating sort order - if (!(PARAM_ASC_SORT_VALUE.equals(pageParams.getSort()) || PARAM_DESC_SORT_VALUE.equals(pageParams.getSort()))) { - throw new IllegalArgumentException("value of sort can either be asc or desc"); - } // Next Token in the request will be validated by the IndexStrategyTokenParser itself. if (Objects.nonNull(pageParams.getRequestedToken())) { IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken()); } - return pageParams; } - @Override - protected Table buildTable( - final RestRequest request, - final Map indicesSettings, - final Map indicesHealths, - final Map indicesStats, - final Map indicesMetadatas, - final String[] indicesToBeQueried, - final PageToken paginatedQueryResponse - ) { - final String healthParam = request.param("health"); - final Table table = getTableWithHeader(request, paginatedQueryResponse); - for (String indexName : indicesToBeQueried) { - if (indicesSettings.containsKey(indexName) == false) { - continue; - } - buildRow(indicesSettings, indicesHealths, indicesStats, indicesMetadatas, healthParam, indexName, table); - } - return table; + protected int defaultPageSize() { + return DEFAULT_LIST_INDICES_PAGE_SIZE_STRING; } @Override @@ -108,12 +79,26 @@ protected IndexPaginationStrategy getPaginationStrategy(ClusterStateResponse clu return new IndexPaginationStrategy(pageParams, clusterStateResponse.getState()); } + // Public for testing @Override - protected PageToken getPageToken(IndexPaginationStrategy paginationStrategy) { - return paginationStrategy.getResponseToken(); - } + public Iterator> getTableIterator(String[] indices, Map indexSettingsMap) { + return new Iterator<>() { + int indexPos = 0; + + @Override + public boolean hasNext() { + while (indexPos < indices.length && indexSettingsMap.containsKey(indices[indexPos]) == false) { + indexPos++; + } + return indexPos < indices.length; + } - protected String[] getIndicesToBeQueried(String[] indices, IndexPaginationStrategy paginationStrategy) { - return paginationStrategy.getRequestedEntities().toArray(new String[0]); + @Override + public Tuple next() { + Tuple tuple = new Tuple<>(indices[indexPos], indexSettingsMap.get(indices[indexPos])); + indexPos++; + return tuple; + } + }; } } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index 7055c51059e58..f9b17bbda94bb 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -32,6 +32,7 @@ package org.opensearch.rest.action.cat; +import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexStats; @@ -47,13 +48,17 @@ import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; +import org.opensearch.rest.action.list.RestIndicesListAction; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; @@ -63,13 +68,14 @@ public class RestIndicesActionTests extends OpenSearchTestCase { - public void testBuildTable() { - final int numIndices = randomIntBetween(3, 20); - final Map indicesSettings = new LinkedHashMap<>(); - final Map indicesMetadatas = new LinkedHashMap<>(); - final Map indicesHealths = new LinkedHashMap<>(); - final Map indicesStats = new LinkedHashMap<>(); + final Map indicesSettings = new LinkedHashMap<>(); + final Map indicesMetadatas = new LinkedHashMap<>(); + final Map indicesHealths = new LinkedHashMap<>(); + final Map indicesStats = new LinkedHashMap<>(); + @Before + public void setup() { + final int numIndices = randomIntBetween(3, 20); for (int i = 0; i < numIndices; i++) { String indexName = "index-" + i; @@ -136,7 +142,9 @@ public void testBuildTable() { } } } + } + public void testBuildTable() { final RestIndicesAction action = new RestIndicesAction(); final Table table = action.buildTable( new FakeRestRequest(), @@ -144,11 +152,49 @@ public void testBuildTable() { indicesHealths, indicesStats, indicesMetadatas, - null, + action.getTableIterator(new String[0], indicesSettings), null ); // now, verify the table is correct + assertNotNull(table); + + assertTableHeaders(table); + + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size())); + assertTableRows(table); + } + + public void testBuildPaginatedTable() { + final RestIndicesAction action = new RestIndicesAction(); + final RestIndicesListAction indicesListAction = new RestIndicesListAction(); + List indicesList = new ArrayList<>(indicesMetadatas.keySet()); + // Using half of the indices from metadata list for a page + String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size()/2).toArray(new String[0]); + PageToken pageToken = new PageToken("foo", "indices"); + final Table table = action.buildTable( + new FakeRestRequest(), + indicesSettings, + indicesHealths, + indicesStats, + indicesMetadatas, + indicesListAction.getTableIterator(indicesToBeQueried, indicesSettings), + pageToken + ); + + // verifying table + assertNotNull(table); + assertTableHeaders(table); + assertNotNull(table.getPageToken()); + assertEquals(pageToken.getNextToken(), table.getPageToken().getNextToken()); + assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity()); + + // Table should only contain the indices present in indicesToBeQueried + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size()/2)); + assertTableRows(table); + } + + private void assertTableHeaders(Table table) { List headers = table.getHeaders(); assertThat(headers.get(0).value, equalTo("health")); assertThat(headers.get(1).value, equalTo("status")); @@ -156,9 +202,10 @@ public void testBuildTable() { assertThat(headers.get(3).value, equalTo("uuid")); assertThat(headers.get(4).value, equalTo("pri")); assertThat(headers.get(5).value, equalTo("rep")); + } + private void assertTableRows(Table table) { final List> rows = table.getRows(); - assertThat(rows.size(), equalTo(indicesMetadatas.size())); for (final List row : rows) { final String indexName = (String) row.get(2).value; diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java index 8183cb1d3b910..a82e563d70273 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestTableTests.java @@ -37,6 +37,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.rest.AbstractRestChannel; import org.opensearch.rest.RestResponse; +import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; import org.junit.Before; @@ -64,9 +65,14 @@ public class RestTableTests extends OpenSearchTestCase { private static final String ACCEPT = "Accept"; private static final String TEXT_PLAIN = "text/plain; charset=UTF-8"; private static final String TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\n"; + private static final String PAGINATED_TEXT_TABLE_BODY = "foo foo foo foo foo foo foo foo\nnext_token foo\n"; private static final String JSON_TABLE_BODY = "[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]"; + private static final String PAGINATED_JSON_TABLE_BODY = + "{\"next_token\":\"foo\",\"entities\":[{\"bulk.foo\":\"foo\",\"bulk.bar\":\"foo\",\"aliasedBulk\":\"foo\"," + + "\"aliasedSecondBulk\":\"foo\",\"unmatched\":\"foo\"," + + "\"invalidAliasesBulk\":\"foo\",\"timestamp\":\"foo\",\"epoch\":\"foo\"}]}"; private static final String YAML_TABLE_BODY = "---\n" + "- bulk.foo: \"foo\"\n" + " bulk.bar: \"foo\"\n" @@ -76,6 +82,17 @@ public class RestTableTests extends OpenSearchTestCase { + " invalidAliasesBulk: \"foo\"\n" + " timestamp: \"foo\"\n" + " epoch: \"foo\"\n"; + private static final String PAGINATED_YAML_TABLE_BODY = "---\n" + + "next_token: \"foo\"\n" + + "entities:\n" + + "- bulk.foo: \"foo\"\n" + + " bulk.bar: \"foo\"\n" + + " aliasedBulk: \"foo\"\n" + + " aliasedSecondBulk: \"foo\"\n" + + " unmatched: \"foo\"\n" + + " invalidAliasesBulk: \"foo\"\n" + + " timestamp: \"foo\"\n" + + " epoch: \"foo\"\n"; private Table table; private FakeRestRequest restRequest; @@ -83,20 +100,7 @@ public class RestTableTests extends OpenSearchTestCase { public void setup() { restRequest = new FakeRestRequest(); table = new Table(); - table.startHeaders(); - table.addCell("bulk.foo", "alias:f;desc:foo"); - table.addCell("bulk.bar", "alias:b;desc:bar"); - // should be matched as well due to the aliases - table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); - table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); - // no match - table.addCell("unmatched", "alias:un.matched;desc:bar"); - // invalid alias - table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); - // timestamp - table.addCell("timestamp", "alias:ts"); - table.addCell("epoch", "alias:t"); - table.endHeaders(); + addHeaders(table); } public void testThatDisplayHeadersSupportWildcards() throws Exception { @@ -121,10 +125,28 @@ public void testThatWeUseTheAcceptHeaderJson() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), APPLICATION_JSON, JSON_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderJsonForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)), + APPLICATION_JSON, + PAGINATED_JSON_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderYaml() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), APPLICATION_YAML, YAML_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderYamlForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)), + APPLICATION_YAML, + PAGINATED_YAML_TABLE_BODY, + getPaginatedTable() + ); + } + public void testThatWeUseTheAcceptHeaderSmile() throws Exception { assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)), APPLICATION_SMILE); } @@ -137,6 +159,15 @@ public void testThatWeUseTheAcceptHeaderText() throws Exception { assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), TEXT_PLAIN, TEXT_TABLE_BODY); } + public void testThatWeUseTheAcceptHeaderTextForPaginatedTable() throws Exception { + assertResponse( + Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)), + TEXT_PLAIN, + PAGINATED_TEXT_TABLE_BODY, + getPaginatedTable() + ); + } + public void testIgnoreContentType() throws Exception { assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), TEXT_PLAIN, TEXT_TABLE_BODY); } @@ -261,6 +292,10 @@ public void testMultiSort() { } private RestResponse assertResponseContentType(Map> headers, String mediaType) throws Exception { + return assertResponseContentType(headers, mediaType, table); + } + + private RestResponse assertResponseContentType(Map> headers, String mediaType, Table table) throws Exception { FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build(); table.startRow(); table.addCell("foo"); @@ -282,7 +317,11 @@ public void sendResponse(RestResponse response) {} } private void assertResponse(Map> headers, String mediaType, String body) throws Exception { - RestResponse response = assertResponseContentType(headers, mediaType); + assertResponse(headers, mediaType, body, table); + } + + private void assertResponse(Map> headers, String mediaType, String body, Table table) throws Exception { + RestResponse response = assertResponseContentType(headers, mediaType, table); assertThat(response.content().utf8ToString(), equalTo(body)); } @@ -294,4 +333,28 @@ private List getHeaderNames(List headers) { return headerNames; } + + private Table getPaginatedTable() { + PageToken pageToken = new PageToken("foo", "entities"); + Table paginatedTable = new Table(pageToken); + addHeaders(paginatedTable); + return paginatedTable; + } + + private void addHeaders(Table table) { + table.startHeaders(); + table.addCell("bulk.foo", "alias:f;desc:foo"); + table.addCell("bulk.bar", "alias:b;desc:bar"); + // should be matched as well due to the aliases + table.addCell("aliasedBulk", "alias:bulkWhatever;desc:bar"); + table.addCell("aliasedSecondBulk", "alias:foobar,bulkolicious,bulkotastic;desc:bar"); + // no match + table.addCell("unmatched", "alias:un.matched;desc:bar"); + // invalid alias + table.addCell("invalidAliasesBulk", "alias:,,,;desc:bar"); + // timestamp + table.addCell("timestamp", "alias:ts"); + table.addCell("epoch", "alias:t"); + table.endHeaders(); + } } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java index 575874cbe6a47..5b0324ca20285 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java @@ -21,15 +21,20 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.List; +import java.util.Objects; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; +import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; +import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; public class IndexPaginationStrategyTests extends OpenSearchTestCase { - public void testRetrieveAllIndicesInAscendingOrder() { + public void testRetrieveAllIndicesWithVaryingPageSize() { List indexNumberList = new ArrayList<>(); final int totalIndices = 100; for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { @@ -42,146 +47,232 @@ public void testRetrieveAllIndicesInAscendingOrder() { // Checking pagination response for different pageSizes, which has a mix of even and odd numbers // to ensure number of indices in last page is not always equal to pageSize. List pageSizeList = List.of(1, 6, 10, 13); - for (int pageSize : pageSizeList) { - String requestedToken = null; - int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); - for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { - PageParams pageParams = new PageParams(requestedToken, "asc", pageSize); - IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - if (pageNumber < totalPagesToFetch) { - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); - } else { - assertNull(paginationStrategy.getResponseToken().getNextToken()); + List sortOrderList = List.of(PARAM_ASC_SORT_VALUE, PARAM_DESC_SORT_VALUE); + for (String sortOrder : sortOrderList) { + for (int pageSize : pageSizeList) { + String requestedToken = null; + int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); + int indicesRemaining = totalIndices; + for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { + PageParams pageParams = new PageParams(requestedToken, sortOrder, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + if (pageNumber < totalPagesToFetch) { + assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + } else { + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // Asserting all the indices received + int responseItr = 0; + if (PARAM_ASC_SORT_VALUE.equals(sortOrder)) { + for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(100, pageNumber * pageSize); indexNumber++) { + assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } else { + int endIndexNumberForPage = Math.max(indicesRemaining - pageSize, 0); + for (; indicesRemaining > endIndexNumberForPage; indicesRemaining--) { + assertEquals("test-index-" + indicesRemaining, paginationStrategy.getRequestedEntities().get(responseItr)); + responseItr++; + } + } + assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); } - requestedToken = paginationStrategy.getResponseToken().getNextToken(); - // Asserting all the indices received - int responseItr = 0; - for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(100, pageNumber * pageSize); indexNumber++) { - assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); - responseItr++; + } + } + } + + public void testRetrieveAllIndicesInAscOrderWhileIndicesGetCreatedAndDeleted() { + List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); + final int totalIndices = 100; + final int numIndicesToDelete = 10; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); + for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { + indexNumberList.add(indexNumber); + } + ClusterState clusterState = getRandomClusterState(indexNumberList); + + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_ASC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(20, indexNumberList.size()); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); } - assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); + } + } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), 5); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices + numIndicesToCreate - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices + numIndicesToCreate - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // all the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertTrue(indicesFetched.contains("test-index-" + indexNumber)); } } - public void testRetrieveAllIndicesInDescendingOrder() { + public void testRetrieveAllIndicesInDescOrderWhileIndicesGetCreatedAndDeleted() { List indexNumberList = new ArrayList<>(); + List deletedIndices = new ArrayList<>(); final int totalIndices = 100; + final int numIndicesToDelete = 9; + final int numIndicesToCreate = 5; + List indicesFetched = new ArrayList<>(); for (int indexNumber = 1; indexNumber <= 100; indexNumber++) { indexNumberList.add(indexNumber); } - // creating a cluster state with 100 indices - Collections.shuffle(indexNumberList, getRandom()); ClusterState clusterState = getRandomClusterState(indexNumberList); - // Checking pagination response for different pageSizes, which has a mix of even and odd numbers - // to ensure number of indices in last page is not always equal to pageSize. - List pageSizeList = List.of(1, 6, 10, 13); - for (int pageSize : pageSizeList) { - String requestedToken = null; - int totalPagesToFetch = (int) Math.ceil(totalIndices / (pageSize * 1.0)); - int startIndexNumber = totalIndices; - for (int pageNumber = 1; pageNumber <= totalPagesToFetch; pageNumber++) { - PageParams pageParams = new PageParams(requestedToken, "desc", pageSize); - IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - if (pageNumber < totalPagesToFetch) { - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); - } else { - assertNull(paginationStrategy.getResponseToken().getNextToken()); + int pageSize = 6; + String requestedToken = null; + int numPages = 0; + do { + numPages++; + PageParams pageParams = new PageParams(requestedToken, PARAM_DESC_SORT_VALUE, pageSize); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertNotNull(paginationStrategy); + assertNotNull(paginationStrategy.getResponseToken()); + requestedToken = paginationStrategy.getResponseToken().getNextToken(); + // randomly deleting 10 indices after 3rd call + if (numPages == 3) { + deletedIndices = indexNumberList.subList(0, 80); + Collections.shuffle(deletedIndices, getRandom()); + for (int pos = 0; pos < numIndicesToDelete; pos++) { + clusterState = deleteIndexFromClusterState(clusterState, deletedIndices.get(pos)); } - requestedToken = paginationStrategy.getResponseToken().getNextToken(); - // Asserting all the indices received - int responseItr = 0; - int endIndexNumberForPage = Math.max(startIndexNumber - pageSize, 0); - for (; startIndexNumber > endIndexNumberForPage; startIndexNumber--) { - assertEquals("test-index-" + startIndexNumber, paginationStrategy.getRequestedEntities().get(responseItr)); - responseItr++; + } + // creating 5 indices after 5th call + if (numPages == 5) { + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + clusterState = addIndexToClusterState(clusterState, indexNumber); } - assertEquals(responseItr, paginationStrategy.getRequestedEntities().size()); } + if (requestedToken == null) { + assertEquals(paginationStrategy.getRequestedEntities().size(), (totalIndices - numIndicesToDelete) % pageSize); + } else { + assertEquals(paginationStrategy.getRequestedEntities().size(), pageSize); + } + + indicesFetched.addAll(paginationStrategy.getRequestedEntities()); + } while (Objects.nonNull(requestedToken)); + + assertEquals((int) Math.ceil((double) (totalIndices - numIndicesToDelete) / pageSize), numPages); + assertEquals(totalIndices - numIndicesToDelete, indicesFetched.size()); + + // none of the deleted index should appear in the list of fetched indices + for (int deletedIndexPos = 0; deletedIndexPos < numIndicesToDelete; deletedIndexPos++) { + assertFalse(indicesFetched.contains("test-index-" + deletedIndices.get(deletedIndexPos))); + } + + // none of the newly created indices should be present in the list of fetched indices + for (int indexNumber = totalIndices + 1; indexNumber <= totalIndices + numIndicesToCreate; indexNumber++) { + assertFalse(indicesFetched.contains("test-index-" + indexNumber)); } } - public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetween() { + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedAscOrder() { // Query1 with 4 indices in clusterState (test-index1,2,3,4) ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - PageParams pageParams = new PageParams(null, "asc", 1); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Adding index5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); - // Deleting test-index-2 which has already been displayed, still test-index-2 should get displayed + // Deleting test-index-2 which has already been displayed, still test-index-3 should get displayed clusterState = deleteIndexFromClusterState(clusterState, 2); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); + // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query // instead test-index-5 should now get displayed. clusterState = deleteIndexFromClusterState(clusterState, 4); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, false); assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); - assertNull(paginationStrategy.getResponseToken().getNextToken()); + } - public void testRetrieveAllIndicesWhenIndicesGetDeletedAndCreatedInBetweenWithDescOrder() { + public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedDescOrder() { // Query1 with 4 indices in clusterState (test-index1,2,3,4). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4)); - PageParams pageParams = new PageParams(null, "desc", 1); + PageParams pageParams = new PageParams(null, PARAM_DESC_SORT_VALUE, 1); IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // adding test-index-5 to clusterState, before executing next query. clusterState = addIndexToClusterState(clusterState, 5); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-3 which has already been displayed, still index2 should get displayed. clusterState = deleteIndexFromClusterState(clusterState, 3); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(1, paginationStrategy.getRequestedEntities().size()); + assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); - assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Deleting test-index-1 which is not yet displayed which otherwise should have been displayed in the following query. clusterState = deleteIndexFromClusterState(clusterState, 1); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "desc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_DESC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); - assertEquals(0, paginationStrategy.getRequestedEntities().size()); - assertNull(paginationStrategy.getResponseToken().getNextToken()); + assertPaginationResult(paginationStrategy, 0, false); } - public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() { + public void testRetrieveIndicesWithMultipleDeletionsAtOnceAscOrder() { // Query1 with 5 indices in clusterState (test-index1,2,3,4,5). ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5)); - PageParams pageParams = new PageParams(null, "asc", 1); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 1); IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); assertEquals(1, paginationStrategy.getRequestedEntities().size()); assertEquals("test-index-1", paginationStrategy.getRequestedEntities().get(0)); assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // executing next query without any changes to clusterState - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); assertEquals(1, paginationStrategy.getRequestedEntities().size()); assertEquals("test-index-2", paginationStrategy.getRequestedEntities().get(0)); @@ -191,20 +282,34 @@ public void testRetrieveAllIndicesWhenMultipleIndicesGetDeletedInBetweenAtOnce() clusterState = deleteIndexFromClusterState(clusterState, 1); clusterState = deleteIndexFromClusterState(clusterState, 2); clusterState = deleteIndexFromClusterState(clusterState, 3); - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); assertEquals(1, paginationStrategy.getRequestedEntities().size()); assertEquals("test-index-4", paginationStrategy.getRequestedEntities().get(0)); assertNotNull(paginationStrategy.getResponseToken().getNextToken()); // Executing the last query without any further change. Should result in test-index-5 and nextToken as null. - pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), "asc", 1); + pageParams = new PageParams(paginationStrategy.getResponseToken().getNextToken(), PARAM_ASC_SORT_VALUE, 1); paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); assertEquals(1, paginationStrategy.getRequestedEntities().size()); assertEquals("test-index-5", paginationStrategy.getRequestedEntities().get(0)); assertNull(paginationStrategy.getResponseToken().getNextToken()); } + public void testRetrieveIndicesWithTokenModifiedToQueryBeyondTotal() { + ClusterState clusterState = getRandomClusterState(List.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); + PageParams pageParams = new PageParams(null, PARAM_ASC_SORT_VALUE, 10); + IndexPaginationStrategy paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(10, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + // creating a token with last sent index as test-index-10 + String token = clusterState.metadata().indices().get("test-index-10").getCreationDate() + "|" + "test-index-10"; + pageParams = new PageParams(Base64.getEncoder().encodeToString(token.getBytes(UTF_8)), PARAM_ASC_SORT_VALUE, 10); + paginationStrategy = new IndexPaginationStrategy(pageParams, clusterState); + assertEquals(0, paginationStrategy.getRequestedEntities().size()); + assertNull(paginationStrategy.getResponseToken().getNextToken()); + } + public void testCreatingIndexStrategyPageTokenWithRequestedTokenNull() { try { new IndexPaginationStrategy.IndexStrategyToken(null); @@ -282,4 +387,11 @@ private ClusterState deleteIndexFromClusterState(ClusterState clusterState, int .build(); } + private void assertPaginationResult(IndexPaginationStrategy paginationStrategy, int expectedEntities, boolean tokenExpected) { + assertNotNull(paginationStrategy); + assertEquals(expectedEntities, paginationStrategy.getRequestedEntities().size()); + assertNotNull(paginationStrategy.getResponseToken()); + assertEquals(tokenExpected, Objects.nonNull(paginationStrategy.getResponseToken().getNextToken())); + } + } From d98cc86a3146fea13e0b41bd498126477d384fae Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Wed, 25 Sep 2024 09:44:33 +0530 Subject: [PATCH 09/12] Applying spotless formatting Signed-off-by: Harsh Garg --- .../opensearch/rest/action/cat/RestSnapshotAction.java | 2 +- .../org/opensearch/rest/action/cat/RestTasksAction.java | 2 +- .../opensearch/rest/action/cat/RestTemplatesAction.java | 2 +- .../rest/action/list/RestIndicesListAction.java | 2 +- .../java/org/opensearch/rest/BaseRestHandlerTests.java | 2 +- .../rest/action/cat/RestIndicesActionTests.java | 7 +++---- .../rest/pagination/IndexPaginationStrategyTests.java | 8 +++++--- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java index fb07a106528f1..f05a84d9b2aa2 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestSnapshotAction.java @@ -96,7 +96,7 @@ public RestResponse buildResponse(GetSnapshotsResponse getSnapshotsResponse) thr } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/snapshots/{repository}\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java index d83d2438dbb99..560b88787ae09 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTasksAction.java @@ -85,7 +85,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/tasks\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java index fafa15820e030..0e9ad8760d4b8 100644 --- a/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java +++ b/server/src/main/java/org/opensearch/rest/action/cat/RestTemplatesAction.java @@ -72,7 +72,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_cat/templates\n"); } diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index 7964d5f3dcdee..283f51d992d57 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -46,7 +46,7 @@ public String getName() { } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { sb.append("/_list/indices\n"); sb.append("/_list/indices/{index}\n"); } diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java index 18c9e987edc3f..45653e9d8e4d6 100644 --- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java +++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java @@ -247,7 +247,7 @@ protected RestChannelConsumer doCatRequest(RestRequest request, NodeClient clien } @Override - public void documentation(StringBuilder sb) { + protected void documentation(StringBuilder sb) { } diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java index f9b17bbda94bb..1d1b509ae94e5 100644 --- a/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/opensearch/rest/action/cat/RestIndicesActionTests.java @@ -32,7 +32,6 @@ package org.opensearch.rest.action.cat; -import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.admin.indices.stats.CommonStats; import org.opensearch.action.admin.indices.stats.IndexStats; @@ -52,13 +51,13 @@ import org.opensearch.rest.pagination.PageToken; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.FakeRestRequest; +import org.junit.Before; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; @@ -170,7 +169,7 @@ public void testBuildPaginatedTable() { final RestIndicesListAction indicesListAction = new RestIndicesListAction(); List indicesList = new ArrayList<>(indicesMetadatas.keySet()); // Using half of the indices from metadata list for a page - String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size()/2).toArray(new String[0]); + String[] indicesToBeQueried = indicesList.subList(0, indicesMetadatas.size() / 2).toArray(new String[0]); PageToken pageToken = new PageToken("foo", "indices"); final Table table = action.buildTable( new FakeRestRequest(), @@ -190,7 +189,7 @@ public void testBuildPaginatedTable() { assertEquals(pageToken.getPaginatedEntity(), table.getPageToken().getPaginatedEntity()); // Table should only contain the indices present in indicesToBeQueried - assertThat(table.getRows().size(), equalTo(indicesMetadatas.size()/2)); + assertThat(table.getRows().size(), equalTo(indicesMetadatas.size() / 2)); assertTableRows(table); } diff --git a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java index 5b0324ca20285..01464b489e26e 100644 --- a/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java +++ b/server/src/test/java/org/opensearch/rest/pagination/IndexPaginationStrategyTests.java @@ -28,9 +28,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE; -import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; import static org.opensearch.rest.pagination.PageParams.PARAM_ASC_SORT_VALUE; import static org.opensearch.rest.pagination.PageParams.PARAM_DESC_SORT_VALUE; +import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom; public class IndexPaginationStrategyTests extends OpenSearchTestCase { @@ -65,7 +65,10 @@ public void testRetrieveAllIndicesWithVaryingPageSize() { // Asserting all the indices received int responseItr = 0; if (PARAM_ASC_SORT_VALUE.equals(sortOrder)) { - for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min(100, pageNumber * pageSize); indexNumber++) { + for (int indexNumber = (pageNumber - 1) * pageSize; indexNumber < Math.min( + 100, + pageNumber * pageSize + ); indexNumber++) { assertEquals("test-index-" + (indexNumber + 1), paginationStrategy.getRequestedEntities().get(responseItr)); responseItr++; } @@ -222,7 +225,6 @@ public void testRetrieveIndicesWithSizeOneAndCurrentIndexGetsDeletedAscOrder() { assertPaginationResult(paginationStrategy, 1, true); assertEquals("test-index-3", paginationStrategy.getRequestedEntities().get(0)); - // Deleting test-index-4 which is not yet displayed which otherwise should have been displayed in the following query // instead test-index-5 should now get displayed. clusterState = deleteIndexFromClusterState(clusterState, 4); From 9a4d19aed43c431fe590e3fdb96026f676713a42 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Sun, 29 Sep 2024 20:12:49 +0530 Subject: [PATCH 10/12] Adding TODO to avoid multiple validations of token Signed-off-by: Harsh Garg --- .../rest/action/list/RestIndicesListAction.java | 12 ++++++------ .../rest/pagination/IndexPaginationStrategy.java | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java index 283f51d992d57..ad5c58c86ce90 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/RestIndicesListAction.java @@ -32,8 +32,8 @@ */ public class RestIndicesListAction extends RestIndicesAction { - protected static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING = 5000; - protected static final int DEFAULT_LIST_INDICES_PAGE_SIZE_STRING = 500; + private static final int MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE = 5000; + private static final int DEFAULT_LIST_INDICES_PAGE_SIZE = 500; @Override public List routes() { @@ -60,10 +60,10 @@ public boolean isActionPaginated() { protected PageParams validateAndGetPageParams(RestRequest restRequest) { PageParams pageParams = super.validateAndGetPageParams(restRequest); // validate max supported pageSize - if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING) { - throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE_STRING + "]"); + if (pageParams.getSize() > MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE) { + throw new IllegalArgumentException("size should be less than [" + MAX_SUPPORTED_LIST_INDICES_PAGE_SIZE + "]"); } - // Next Token in the request will be validated by the IndexStrategyTokenParser itself. + // Next Token in the request will be validated by the IndexStrategyToken itself. if (Objects.nonNull(pageParams.getRequestedToken())) { IndexPaginationStrategy.IndexStrategyToken.validateIndexStrategyToken(pageParams.getRequestedToken()); } @@ -71,7 +71,7 @@ protected PageParams validateAndGetPageParams(RestRequest restRequest) { } protected int defaultPageSize() { - return DEFAULT_LIST_INDICES_PAGE_SIZE_STRING; + return DEFAULT_LIST_INDICES_PAGE_SIZE; } @Override diff --git a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java index 3a84226dc85cf..f89ab14e4b24d 100644 --- a/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java +++ b/server/src/main/java/org/opensearch/rest/pagination/IndexPaginationStrategy.java @@ -29,7 +29,7 @@ * @opensearch.internal */ public class IndexPaginationStrategy implements PaginationStrategy { - private static final String DEFAULT_INDICES_PAGINATED_ELEMENT = "indices"; + private static final String DEFAULT_INDICES_PAGINATED_ENTITY = "indices"; private static final Comparator ASC_COMPARATOR = (metadata1, metadata2) -> { if (metadata1.getCreationDate() == metadata2.getCreationDate()) { @@ -96,11 +96,11 @@ private List getMetadataSubList(List sortedIndices private PageToken getResponseToken(final int pageSize, final int totalIndices, IndexMetadata lastIndex) { if (totalIndices <= pageSize) { - return new PageToken(null, DEFAULT_INDICES_PAGINATED_ELEMENT); + return new PageToken(null, DEFAULT_INDICES_PAGINATED_ENTITY); } return new PageToken( new IndexStrategyToken(lastIndex.getCreationDate(), lastIndex.getIndex().getName()).generateEncryptedToken(), - DEFAULT_INDICES_PAGINATED_ELEMENT + DEFAULT_INDICES_PAGINATED_ENTITY ); } @@ -117,8 +117,7 @@ public List getRequestedEntities() { /** * TokenParser to be used by {@link IndexPaginationStrategy}. - * Token would look like: IndexNumberToStartTheNextPageFrom + | + CreationTimeOfLastRespondedIndex + | + - * QueryStartTime + | + NameOfLastRespondedIndex + * Token would look like: CreationTimeOfLastRespondedIndex + | + NameOfLastRespondedIndex */ public static class IndexStrategyToken { @@ -140,6 +139,7 @@ public static class IndexStrategyToken { private final String lastIndexName; public IndexStrategyToken(String requestedTokenString) { + // TODO: Avoid validating the requested token multiple times while calling from Rest and/or Transport layer. validateIndexStrategyToken(requestedTokenString); String decryptedToken = PaginationStrategy.decryptStringToken(requestedTokenString); final String[] decryptedTokenElements = decryptedToken.split(SPLIT_REGEX); From e7eb940ee0022e84b07eeca44ead4082f42efc21 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Sun, 29 Sep 2024 20:16:39 +0530 Subject: [PATCH 11/12] Adding changeLog Signed-off-by: Harsh Garg --- CHANGELOG.md | 1 + .../org/opensearch/rest/action/list/AbstractListAction.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07637eaae3306..b628e9277959d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967)) - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430)) - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923)) +- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718)) ### Dependencies - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578)) diff --git a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java index 6b34a7585c658..f3d6d6653a550 100644 --- a/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java +++ b/server/src/main/java/org/opensearch/rest/action/list/AbstractListAction.java @@ -21,7 +21,8 @@ /** * Base Transport action class for _list API. - * + * Serves as a base class for APIs wanting to support pagination. + * Existing _cat APIs can refer {@link org.opensearch.rest.action.cat.RestIndicesAction}. * @opensearch.api */ public abstract class AbstractListAction extends AbstractCatAction { From c8261d4ee77f3cfc48372d7706f6cad13e991726 Mon Sep 17 00:00:00 2001 From: Harsh Garg Date: Sun, 29 Sep 2024 21:52:07 +0530 Subject: [PATCH 12/12] Retry Build Signed-off-by: Harsh Garg