From 26ec5fc2aae45bd74eb8993e055546589f3a3731 Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Wed, 1 Feb 2023 18:37:33 +0530 Subject: [PATCH 1/9] [Weighted Routing] Support for fail open stats (#6041) * Support for fail open stats Signed-off-by: Anshu Agarwal --- .../search/SearchWeightedRoutingIT.java | 112 ++++++++++++++++++ .../admin/cluster/node/stats/NodeStats.java | 24 +++- .../cluster/node/stats/NodesStatsRequest.java | 3 +- .../node/stats/TransportNodesStatsAction.java | 3 +- .../stats/TransportClusterStatsAction.java | 1 + .../routing/FailAwareWeightedRouting.java | 6 + .../cluster/routing/WeightedRoutingStats.java | 84 +++++++++++++ .../java/org/opensearch/node/NodeService.java | 7 +- .../cluster/node/stats/NodeStatsTests.java | 18 ++- .../opensearch/cluster/DiskUsageTests.java | 6 + .../MockInternalClusterInfoService.java | 3 +- .../opensearch/test/InternalTestCluster.java | 1 + 12 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java diff --git a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java index 279b07344b3f9..a1ccc3e3a1630 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/SearchWeightedRoutingIT.java @@ -21,12 +21,16 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.PreferenceBasedSearchNotAllowedException; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.WeightedRouting; +import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.search.stats.SearchStats; +import org.opensearch.index.shard.ShardId; import org.opensearch.plugins.Plugin; import org.opensearch.rest.RestStatus; import org.opensearch.search.aggregations.Aggregations; @@ -869,4 +873,112 @@ public void testPreferenceSearchWithWeightedRouting() throws Exception { assertEquals(RestStatus.OK.getStatus(), searchResponse.status().getStatus()); } + /** + * Shard routing request is served by data nodes in az with weight set as 0, + * in case shard copies are not available in other azs.(with fail open enabled) + * This is tested by setting up a 3 node cluster with one data node per az. + * Weighted shard routing weight is set as 0 for az-c. + * Indices are created with one replica copy and network disruption is introduced, + * which makes data node in zone-a unresponsive. + * Since there are two copies of a shard, there can be few shards for which copy doesn't exist in zone b. + * Assertions are put to make sure such shard search requests are served by data node in zone c. + * Asserts on fail open stats which captures number of times fail open is executed + * @throws IOException throws exception + */ + public void testWeightedRoutingFailOpenStats() throws Exception { + Settings commonSettings = Settings.builder() + .put("cluster.routing.allocation.awareness.attributes", "zone") + .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c") + .put("cluster.routing.weighted.fail_open", true) + .build(); + + int nodeCountPerAZ = 1; + Map> nodeMap = setupCluster(nodeCountPerAZ, commonSettings); + + int numShards = 10; + int numReplicas = 1; + setUpIndexing(numShards, numReplicas); + + logger.info("--> setting shard routing weights for weighted round robin"); + Map weights = Map.of("a", 1.0, "b", 1.0, "c", 0.0); + setShardRoutingWeights(weights); + WeightedRoutingStats.getInstance().resetFailOpenCount(); + + logger.info("--> creating network partition disruption"); + final String clusterManagerNode1 = internalCluster().getClusterManagerName(); + Set nodesInOneSide = Stream.of(clusterManagerNode1, nodeMap.get("b").get(0)).collect(Collectors.toCollection(HashSet::new)); + Set nodesInOtherSide = Stream.of(nodeMap.get("a").get(0)).collect(Collectors.toCollection(HashSet::new)); + + NetworkDisruption networkDisruption = new NetworkDisruption( + new NetworkDisruption.TwoPartitions(nodesInOneSide, nodesInOtherSide), + NetworkDisruption.UNRESPONSIVE + ); + internalCluster().setDisruptionScheme(networkDisruption); + + DiscoveryNodes dataNodes = internalCluster().clusterService().state().nodes(); + + Map nodeIDMap = new HashMap<>(); + for (DiscoveryNode node : dataNodes) { + nodeIDMap.put(node.getName(), node.getId()); + } + + List shardInNodeA = internalCluster().clusterService() + .state() + .getRoutingNodes() + .node(nodeIDMap.get(nodeMap.get("a").get(0))) + .shardsWithState(ShardRoutingState.STARTED); + + List shardInNodeC = internalCluster().clusterService() + .state() + .getRoutingNodes() + .node(nodeIDMap.get(nodeMap.get("c").get(0))) + .shardsWithState(ShardRoutingState.STARTED); + + // fail open will be called for shards in zone-a data node with replica in zone-c data node + Set result = new HashSet<>(); + int failOpenShardCount = 0; + for (ShardRouting shardRouting : shardInNodeA) { + result.add(shardRouting.shardId()); + } + for (ShardRouting shardRouting : shardInNodeC) { + if (result.contains(shardRouting.shardId())) { + failOpenShardCount++; + } + } + + logger.info("--> network disruption is started"); + networkDisruption.startDisrupting(); + + Set hitNodes = new HashSet<>(); + logger.info("--> making search requests"); + + Future response = internalCluster().client(nodeMap.get("b").get(0)) + .prepareSearch("test") + .setSize(100) + .setQuery(QueryBuilders.matchAllQuery()) + .execute(); + + logger.info("--> network disruption is stopped"); + networkDisruption.stopDisrupting(); + + try { + SearchResponse searchResponse = response.get(); + assertEquals(searchResponse.getFailedShards(), 0); + for (int j = 0; j < searchResponse.getHits().getHits().length; j++) { + hitNodes.add(searchResponse.getHits().getAt(j).getShard().getNodeId()); + } + } catch (Exception t) { + fail("search should not fail"); + } + assertSearchInAZ("b"); + assertSearchInAZ("c"); + assertNoSearchInAZ("a"); + + NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().addMetric("weighted_routing").execute().actionGet(); + Map stats = nodeStats.getNodesMap(); + NodeStats nodeStatsC = stats.get(nodeIDMap.get(nodeMap.get("c").get(0))); + assertEquals(failOpenShardCount, nodeStatsC.getWeightedRoutingStats().getFailOpenCount()); + WeightedRoutingStats.getInstance().resetFailOpenCount(); + } + } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index a83a5f71605cf..ce3cea5260966 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -36,6 +36,7 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.StreamInput; @@ -126,6 +127,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private ClusterManagerThrottlingStats clusterManagerThrottlingStats; + @Nullable + private WeightedRoutingStats weightedRoutingStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -162,6 +166,11 @@ public NodeStats(StreamInput in) throws IOException { } else { clusterManagerThrottlingStats = null; } + if (in.getVersion().onOrAfter(Version.CURRENT)) { + weightedRoutingStats = in.readOptionalWriteable(WeightedRoutingStats::new); + } else { + weightedRoutingStats = null; + } } public NodeStats( @@ -184,7 +193,8 @@ public NodeStats( @Nullable IndexingPressureStats indexingPressureStats, @Nullable ShardIndexingPressureStats shardIndexingPressureStats, @Nullable SearchBackpressureStats searchBackpressureStats, - @Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats + @Nullable ClusterManagerThrottlingStats clusterManagerThrottlingStats, + @Nullable WeightedRoutingStats weightedRoutingStats ) { super(node); this.timestamp = timestamp; @@ -206,6 +216,7 @@ public NodeStats( this.shardIndexingPressureStats = shardIndexingPressureStats; this.searchBackpressureStats = searchBackpressureStats; this.clusterManagerThrottlingStats = clusterManagerThrottlingStats; + this.weightedRoutingStats = weightedRoutingStats; } public long getTimestamp() { @@ -325,6 +336,10 @@ public ClusterManagerThrottlingStats getClusterManagerThrottlingStats() { return clusterManagerThrottlingStats; } + public WeightedRoutingStats getWeightedRoutingStats() { + return weightedRoutingStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -356,6 +371,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_6_0)) { out.writeOptionalWriteable(clusterManagerThrottlingStats); } + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeOptionalWriteable(weightedRoutingStats); + } } @Override @@ -434,6 +452,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getClusterManagerThrottlingStats() != null) { getClusterManagerThrottlingStats().toXContent(builder, params); } + if (getWeightedRoutingStats() != null) { + getWeightedRoutingStats().toXContent(builder, params); + } + return builder; } } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 87cb4b2859dc8..be9a416c979c6 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -208,7 +208,8 @@ public enum Metric { INDEXING_PRESSURE("indexing_pressure"), SHARD_INDEXING_PRESSURE("shard_indexing_pressure"), SEARCH_BACKPRESSURE("search_backpressure"), - CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"); + CLUSTER_MANAGER_THROTTLING("cluster_manager_throttling"), + WEIGHTED_ROUTING_STATS("weighted_routing"); private String metricName; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index e3bb8b52144ac..08d46df723e2e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -120,7 +120,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) { NodesStatsRequest.Metric.INDEXING_PRESSURE.containedIn(metrics), NodesStatsRequest.Metric.SHARD_INDEXING_PRESSURE.containedIn(metrics), NodesStatsRequest.Metric.SEARCH_BACKPRESSURE.containedIn(metrics), - NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics) + NodesStatsRequest.Metric.CLUSTER_MANAGER_THROTTLING.containedIn(metrics), + NodesStatsRequest.Metric.WEIGHTED_ROUTING_STATS.containedIn(metrics) ); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java index ff7f67d7bca2b..fc599efa13e68 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -164,6 +164,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq false, false, false, + false, false ); List shardsStats = new ArrayList<>(); diff --git a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java index 0ddaed2157514..f07f4ec86ad8a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java +++ b/server/src/main/java/org/opensearch/cluster/routing/FailAwareWeightedRouting.java @@ -103,6 +103,7 @@ public SearchShardTarget findNext(final SearchShardIterator shardIt, ClusterStat SearchShardTarget nextShard = next; if (canFailOpen(nextShard.getShardId(), exception, clusterState)) { logger.info(() -> new ParameterizedMessage("{}: Fail open executed due to exception", nextShard.getShardId()), exception); + getWeightedRoutingStats().updateFailOpenCount(); break; } next = shardIt.nextOrNull(); @@ -125,6 +126,7 @@ public ShardRouting findNext(final ShardsIterator shardsIt, ClusterState cluster ShardRouting nextShard = next; if (canFailOpen(nextShard.shardId(), exception, clusterState)) { logger.info(() -> new ParameterizedMessage("{}: Fail open executed due to exception", nextShard.shardId()), exception); + getWeightedRoutingStats().updateFailOpenCount(); break; } next = shardsIt.nextOrNull(); @@ -150,4 +152,8 @@ private boolean hasInActiveShardCopies(ClusterState clusterState, ShardId shardI } return false; } + + public WeightedRoutingStats getWeightedRoutingStats() { + return WeightedRoutingStats.getInstance(); + } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java new file mode 100644 index 0000000000000..58ff30c0c491c --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRoutingStats.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing; + +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.ToXContentFragment; +import org.opensearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Captures weighted shard routing stats per node. See {@link WeightedRoutingService} for more details. + * + * @opensearch.internal + */ +public class WeightedRoutingStats implements ToXContentFragment, Writeable { + // number of times fail open has to be executed for search requests + private AtomicInteger failOpenCount; + + private static final WeightedRoutingStats INSTANCE = new WeightedRoutingStats(); + + public static WeightedRoutingStats getInstance() { + return INSTANCE; + } + + private WeightedRoutingStats() { + failOpenCount = new AtomicInteger(0); + } + + public WeightedRoutingStats(StreamInput in) throws IOException { + failOpenCount = new AtomicInteger(in.readInt()); + } + + public void updateFailOpenCount() { + failOpenCount.getAndIncrement(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject("weighted_routing"); + builder.startObject("stats"); + builder.field("fail_open_count", getFailOpenCount()); + builder.endObject(); + builder.endObject(); + return builder; + } + + public int getFailOpenCount() { + return failOpenCount.get(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(failOpenCount.get()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WeightedRoutingStats that = (WeightedRoutingStats) o; + return failOpenCount.equals(that.failOpenCount); + } + + @Override + public int hashCode() { + return Objects.hash(failOpenCount); + } + + public void resetFailOpenCount() { + failOpenCount.set(0); + } +} diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java index 53b0b3a1751e2..ebceb175380e5 100644 --- a/server/src/main/java/org/opensearch/node/NodeService.java +++ b/server/src/main/java/org/opensearch/node/NodeService.java @@ -32,6 +32,7 @@ package org.opensearch.node; +import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.core.internal.io.IOUtils; import org.opensearch.Build; import org.opensearch.Version; @@ -177,7 +178,8 @@ public NodeStats stats( boolean indexingPressure, boolean shardIndexingPressure, boolean searchBackpressure, - boolean clusterManagerThrottling + boolean clusterManagerThrottling, + boolean weightedRoutingStats ) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) @@ -201,7 +203,8 @@ public NodeStats stats( indexingPressure ? this.indexingPressureService.nodeStats() : null, shardIndexingPressure ? this.indexingPressureService.shardStats(indices) : null, searchBackpressure ? this.searchBackpressureService.nodeStats() : null, - clusterManagerThrottling ? this.clusterService.getClusterManagerService().getThrottlingStats() : null + clusterManagerThrottling ? this.clusterService.getClusterManagerService().getThrottlingStats() : null, + weightedRoutingStats ? WeightedRoutingStats.getInstance() : null ); } diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java index 172acd6a5b9ed..a726a522e547b 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -33,6 +33,7 @@ package org.opensearch.action.admin.cluster.node.stats; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.WeightedRoutingStats; import org.opensearch.cluster.service.ClusterManagerThrottlingStats; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.io.stream.StreamInput; @@ -434,6 +435,15 @@ public void testSerialization() throws IOException { deserializedClusterManagerThrottlingStats.getThrottlingCount("test-task") ); } + + WeightedRoutingStats weightedRoutingStats = nodeStats.getWeightedRoutingStats(); + WeightedRoutingStats deserializedWeightedRoutingStats = deserializedNodeStats.getWeightedRoutingStats(); + if (weightedRoutingStats == null) { + assertNull(deserializedWeightedRoutingStats); + } else { + assertEquals(weightedRoutingStats.getFailOpenCount(), deserializedWeightedRoutingStats.getFailOpenCount()); + + } } } } @@ -711,6 +721,11 @@ public static NodeStats createNodeStats() { clusterManagerThrottlingStats.onThrottle("test-task", randomInt()); } ScriptCacheStats scriptCacheStats = scriptStats != null ? scriptStats.toScriptCacheStats() : null; + + WeightedRoutingStats weightedRoutingStats = null; + weightedRoutingStats = WeightedRoutingStats.getInstance(); + weightedRoutingStats.updateFailOpenCount(); + // TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet return new NodeStats( node, @@ -732,7 +747,8 @@ public static NodeStats createNodeStats() { null, null, null, - clusterManagerThrottlingStats + clusterManagerThrottlingStats, + weightedRoutingStats ); } diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java index 513b16cb744d1..c2c563d177dc7 100644 --- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java @@ -187,6 +187,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -209,6 +210,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ), new NodeStats( @@ -231,6 +233,7 @@ public void testFillDiskUsage() { null, null, null, + null, null ) ); @@ -284,6 +287,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -306,6 +310,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ), new NodeStats( @@ -328,6 +333,7 @@ public void testFillDiskUsageSomeInvalidValues() { null, null, null, + null, null ) ); diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java index 40e23cd0d5856..c47c090714383 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java @@ -116,7 +116,8 @@ List adjustNodesStats(List nodesStats) { nodeStats.getIndexingPressureStats(), nodeStats.getShardIndexingPressureStats(), nodeStats.getSearchBackpressureStats(), - nodeStats.getClusterManagerThrottlingStats() + nodeStats.getClusterManagerThrottlingStats(), + nodeStats.getWeightedRoutingStats() ); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index a929d8e19366d..346657b03c511 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2687,6 +2687,7 @@ public void ensureEstimatedStats() { false, false, false, + false, false ); assertThat( From fd3812c58bd15b75d43dda2787789c58af5caeea Mon Sep 17 00:00:00 2001 From: Anshu Agarwal Date: Wed, 1 Feb 2023 21:32:01 +0530 Subject: [PATCH 2/9] Fix weighted routing stats version (#6126) Signed-off-by: Anshu Agarwal --- .../opensearch/action/admin/cluster/node/stats/NodeStats.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java index ce3cea5260966..b7e0fa9fe435d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java @@ -166,7 +166,7 @@ public NodeStats(StreamInput in) throws IOException { } else { clusterManagerThrottlingStats = null; } - if (in.getVersion().onOrAfter(Version.CURRENT)) { + if (in.getVersion().onOrAfter(Version.V_2_6_0)) { weightedRoutingStats = in.readOptionalWriteable(WeightedRoutingStats::new); } else { weightedRoutingStats = null; @@ -371,7 +371,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_6_0)) { out.writeOptionalWriteable(clusterManagerThrottlingStats); } - if (out.getVersion().onOrAfter(Version.CURRENT)) { + if (out.getVersion().onOrAfter(Version.V_2_6_0)) { out.writeOptionalWriteable(weightedRoutingStats); } } From 6176ddc0ae831095a0e2e8a38b0f475158abef3b Mon Sep 17 00:00:00 2001 From: Sarat Vemulapalli Date: Wed, 1 Feb 2023 10:24:08 -0800 Subject: [PATCH 3/9] [Extensions] Add support for minimum compatible version for extensions with OpenSearch (#6003) * Refactoring extensions config to have real useable values Signed-off-by: Sarat Vemulapalli * Removing werror Signed-off-by: Sarat Vemulapalli * Removing extensions feature flag from gradle run Signed-off-by: Sarat Vemulapalli * Removing description Signed-off-by: Sarat Vemulapalli * Adding MinimumCompatible Version for extensions Signed-off-by: Sarat Vemulapalli * Adding changelog Signed-off-by: Sarat Vemulapalli * Updating changelog Signed-off-by: Sarat Vemulapalli * Addressing comments Signed-off-by: Sarat Vemulapalli * Addressing comments Signed-off-by: Sarat Vemulapalli * Removing debug lines Signed-off-by: Sarat Vemulapalli * Removing another log line Signed-off-by: Sarat Vemulapalli * Updating spotless Signed-off-by: Sarat Vemulapalli --------- Signed-off-by: Sarat Vemulapalli --- CHANGELOG.md | 1 + .../extensions/DiscoveryExtensionNode.java | 33 +++- .../extensions/ExtensionsManager.java | 30 +-- .../extensions/ExtensionsSettings.java | 124 +++--------- .../DiscoveryExtensionNodeTests.java | 52 +++++ .../extensions/ExtensionsManagerTests.java | 178 +++++++----------- ...ExtensionTransportActionsHandlerTests.java | 16 +- .../rest/RestSendToExtensionActionTests.java | 15 +- .../src/test/resources/config/extensions.yml | 18 -- 9 files changed, 184 insertions(+), 283 deletions(-) create mode 100644 server/src/test/java/org/opensearch/extensions/DiscoveryExtensionNodeTests.java delete mode 100644 server/src/test/resources/config/extensions.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d371b56e930b..272f4deefd713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589)) - Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874)) - Changing ExtensionActionRequest streaminput constructor to be public ([#6094](https://github.com/opensearch-project/OpenSearch/pull/6094)) +- Adds support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003)) ### Dependencies - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704)) diff --git a/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java b/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java index 1cf3afd9568e6..7a15be49d86ee 100644 --- a/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java +++ b/server/src/main/java/org/opensearch/extensions/DiscoveryExtensionNode.java @@ -8,6 +8,7 @@ package org.opensearch.extensions; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; @@ -17,7 +18,6 @@ import org.opensearch.common.transport.TransportAddress; import org.opensearch.common.xcontent.ToXContentFragment; import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.plugins.PluginInfo; import java.io.IOException; import java.util.ArrayList; @@ -32,30 +32,28 @@ */ public class DiscoveryExtensionNode extends DiscoveryNode implements Writeable, ToXContentFragment { - private final PluginInfo pluginInfo; + private Version minimumCompatibleVersion; private List dependencies = Collections.emptyList(); public DiscoveryExtensionNode( String name, String id, - String ephemeralId, - String hostName, - String hostAddress, TransportAddress address, Map attributes, Version version, - PluginInfo pluginInfo, + Version minimumCompatibleVersion, List dependencies ) { - super(name, id, ephemeralId, hostName, hostAddress, address, attributes, DiscoveryNodeRole.BUILT_IN_ROLES, version); - this.pluginInfo = pluginInfo; + super(name, id, address, attributes, DiscoveryNodeRole.BUILT_IN_ROLES, version); + this.minimumCompatibleVersion = minimumCompatibleVersion; this.dependencies = dependencies; + validate(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - pluginInfo.writeTo(out); + Version.writeVersion(minimumCompatibleVersion, out); out.writeVInt(dependencies.size()); for (ExtensionDependency dependency : dependencies) { dependency.writeTo(out); @@ -70,7 +68,7 @@ public void writeTo(StreamOutput out) throws IOException { */ public DiscoveryExtensionNode(final StreamInput in) throws IOException { super(in); - this.pluginInfo = new PluginInfo(in); + minimumCompatibleVersion = Version.readVersion(in); int size = in.readVInt(); dependencies = new ArrayList<>(size); for (int i = 0; i < size; i++) { @@ -82,6 +80,10 @@ public List getDependencies() { return dependencies; } + public Version getMinimumCompatibleVersion() { + return minimumCompatibleVersion; + } + public boolean dependenciesContain(ExtensionDependency dependency) { for (ExtensionDependency extensiondependency : this.dependencies) { if (dependency.getUniqueId().equals(extensiondependency.getUniqueId()) @@ -92,6 +94,17 @@ public boolean dependenciesContain(ExtensionDependency dependency) { return false; } + private void validate() { + if (!Version.CURRENT.onOrAfter(minimumCompatibleVersion)) { + throw new OpenSearchException( + "Extension minimumCompatibleVersion: " + + minimumCompatibleVersion + + " is greater than current OpenSearch version: " + + Version.CURRENT + ); + } + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return null; diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index ca65215599891..500d06b3ccc3d 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -28,6 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.client.node.NodeClient; @@ -57,7 +58,6 @@ import org.opensearch.index.IndicesModuleResponse; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.indices.cluster.IndicesClusterStateService; -import org.opensearch.plugins.PluginInfo; import org.opensearch.rest.RestController; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportException; @@ -289,7 +289,7 @@ private void registerRequestHandler() { * Load and populate all extensions */ private void discover() throws IOException { - logger.info("Extensions Config Directory :" + extensionsPath.toString()); + logger.info("Loading extensions : {}", extensionsPath); if (!FileSystemUtils.isAccessibleDirectory(extensionsPath, logger)) { return; } @@ -308,7 +308,7 @@ private void discover() throws IOException { logger.info("Loaded all extensions"); } } else { - logger.info("Extensions.yml file is not present. No extensions will be loaded."); + logger.warn("Extensions.yml file is not present. No extensions will be loaded."); } } @@ -324,27 +324,16 @@ private void loadExtension(Extension extension) throws IOException { DiscoveryExtensionNode discoveryExtensionNode = new DiscoveryExtensionNode( extension.getName(), extension.getUniqueId(), - // placeholder for ephemeral id, will change with POC discovery - extension.getUniqueId(), - extension.getHostName(), - extension.getHostAddress(), new TransportAddress(InetAddress.getByName(extension.getHostAddress()), Integer.parseInt(extension.getPort())), new HashMap(), Version.fromString(extension.getOpensearchVersion()), - new PluginInfo( - extension.getName(), - extension.getDescription(), - extension.getVersion(), - Version.fromString(extension.getOpensearchVersion()), - extension.getJavaVersion(), - extension.getClassName(), - new ArrayList(), - Boolean.parseBoolean(extension.hasNativeController()) - ), + Version.fromString(extension.getMinimumCompatibleVersion()), extension.getDependencies() ); extensionIdMap.put(extension.getUniqueId(), discoveryExtensionNode); logger.info("Loaded extension with uniqueId " + extension.getUniqueId() + ": " + extension); + } catch (OpenSearchException e) { + logger.error("Could not load extension with uniqueId " + extension.getUniqueId() + " due to " + e); } catch (IllegalArgumentException e) { throw e; } @@ -580,16 +569,11 @@ private ExtensionsSettings readFromExtensionsYml(Path filePath) throws IOExcepti new Extension( extensionMap.get("name").toString(), extensionMap.get("uniqueId").toString(), - extensionMap.get("hostName").toString(), extensionMap.get("hostAddress").toString(), extensionMap.get("port").toString(), extensionMap.get("version").toString(), - extensionMap.get("description").toString(), extensionMap.get("opensearchVersion").toString(), - extensionMap.get("javaVersion").toString(), - extensionMap.get("className").toString(), - extensionMap.get("customFolderName").toString(), - extensionMap.get("hasNativeController").toString() + extensionMap.get("minimumCompatibleVersion").toString() ) ); } diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsSettings.java b/server/src/main/java/org/opensearch/extensions/ExtensionsSettings.java index 01c8223075ada..51815a002d1df 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsSettings.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsSettings.java @@ -38,59 +38,39 @@ public static class Extension { private String name; private String uniqueId; - private String hostName; private String hostAddress; private String port; private String version; - private String description; private String opensearchVersion; - private String jvmVersion; - private String className; - private String customFolderName; - private String hasNativeController; + private String minimumCompatibleVersion; private List dependencies = Collections.emptyList(); public Extension( String name, String uniqueId, - String hostName, String hostAddress, String port, String version, - String description, String opensearchVersion, - String jvmVersion, - String className, - String customFolderName, - String hasNativeController + String minimumCompatibleVersion ) { this.name = name; this.uniqueId = uniqueId; - this.hostName = hostName; this.hostAddress = hostAddress; this.port = port; this.version = version; - this.description = description; this.opensearchVersion = opensearchVersion; - this.jvmVersion = jvmVersion; - this.className = className; - this.customFolderName = customFolderName; - this.hasNativeController = hasNativeController; + this.minimumCompatibleVersion = minimumCompatibleVersion; } public Extension() { name = ""; uniqueId = ""; - hostName = ""; hostAddress = ""; port = ""; version = ""; - description = ""; opensearchVersion = ""; - jvmVersion = ""; - className = ""; - customFolderName = ""; - hasNativeController = "false"; + minimumCompatibleVersion = ""; } public String getName() { @@ -109,14 +89,6 @@ public void setUniqueId(String uniqueId) { this.uniqueId = uniqueId; } - public String getHostName() { - return hostName; - } - - public void setHostName(String hostName) { - this.hostName = hostName; - } - public String getHostAddress() { return hostAddress; } @@ -141,43 +113,6 @@ public void setVersion(String version) { this.version = version; } - @Override - public String toString() { - return "Extension [className=" - + className - + ", customFolderName=" - + customFolderName - + ", description=" - + description - + ", hasNativeController=" - + hasNativeController - + ", hostAddress=" - + hostAddress - + ", hostName=" - + hostName - + ", jvmVersion=" - + jvmVersion - + ", name=" - + name - + ", opensearchVersion=" - + opensearchVersion - + ", port=" - + port - + ", uniqueId=" - + uniqueId - + ", version=" - + version - + "]"; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - public String getOpensearchVersion() { return opensearchVersion; } @@ -186,40 +121,35 @@ public void setOpensearchVersion(String opensearchVersion) { this.opensearchVersion = opensearchVersion; } - public String getJavaVersion() { - return jvmVersion; - } - - public void setJavaVersion(String jvmVersion) { - this.jvmVersion = jvmVersion; - } - - public String getClassName() { - return className; - } - - public void setClassName(String className) { - this.className = className; - } - - public String getCustomFolderName() { - return customFolderName; - } - - public void setCustomFolderName(String customFolderName) { - this.customFolderName = customFolderName; + public List getDependencies() { + return dependencies; } - public String hasNativeController() { - return hasNativeController; + public String getMinimumCompatibleVersion() { + return minimumCompatibleVersion; } - public void setHasNativeController(String hasNativeController) { - this.hasNativeController = hasNativeController; + public void setMinimumCompatibleVersion(String minimumCompatibleVersion) { + this.minimumCompatibleVersion = minimumCompatibleVersion; } - public List getDependencies() { - return dependencies; + @Override + public String toString() { + return "Extension [name=" + + name + + ", uniqueId=" + + uniqueId + + ", hostAddress=" + + hostAddress + + ", port=" + + port + + ", version=" + + version + + ", opensearchVersion=" + + opensearchVersion + + ", minimumCompatibleVersion=" + + minimumCompatibleVersion + + "]"; } } diff --git a/server/src/test/java/org/opensearch/extensions/DiscoveryExtensionNodeTests.java b/server/src/test/java/org/opensearch/extensions/DiscoveryExtensionNodeTests.java new file mode 100644 index 0000000000000..578e7503b76a9 --- /dev/null +++ b/server/src/test/java/org/opensearch/extensions/DiscoveryExtensionNodeTests.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.extensions; + +import org.opensearch.OpenSearchException; +import org.opensearch.Version; +import org.opensearch.common.transport.TransportAddress; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; +import java.util.HashMap; + +public class DiscoveryExtensionNodeTests extends OpenSearchTestCase { + + public void testExtensionNode() throws UnknownHostException { + DiscoveryExtensionNode extensionNode = new DiscoveryExtensionNode( + "firstExtension", + "extensionUniqueId1", + new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), + new HashMap(), + Version.fromString("3.0.0"), + // minimum compatible version + Version.fromString("3.0.0"), + Collections.emptyList() + ); + assertTrue(Version.CURRENT.onOrAfter(extensionNode.getMinimumCompatibleVersion())); + } + + public void testIncompatibleExtensionNode() throws UnknownHostException { + expectThrows( + OpenSearchException.class, + () -> new DiscoveryExtensionNode( + "firstExtension", + "extensionUniqueId1", + new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), + new HashMap(), + Version.fromString("3.0.0"), + // minimum compatible version + Version.fromString("3.99.0"), + Collections.emptyList() + ) + ); + } +} diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 5de2113672ca5..44ca2175621e7 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -74,7 +74,6 @@ import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.plugins.PluginInfo; import org.opensearch.rest.RestController; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.IndexSettingsModule; @@ -111,28 +110,18 @@ public class ExtensionsManagerTests extends OpenSearchTestCase { "extensions:", " - name: firstExtension", " uniqueId: uniqueid1", - " hostName: 'myIndependentPluginHost1'", " hostAddress: '127.0.0.0'", " port: '9300'", " version: '0.0.7'", - " description: Fake description 1", " opensearchVersion: '3.0.0'", - " javaVersion: '14'", - " className: fakeClass1", - " customFolderName: fakeFolder1", - " hasNativeController: false", + " minimumCompatibleVersion: '3.0.0'", " - name: secondExtension", " uniqueId: 'uniqueid2'", - " hostName: 'myIndependentPluginHost2'", " hostAddress: '127.0.0.1'", " port: '9301'", " version: '3.14.16'", - " description: Fake description 2", " opensearchVersion: '2.0.0'", - " javaVersion: '17'", - " className: fakeClass2", - " customFolderName: fakeFolder2", - " hasNativeController: true", + " minimumCompatibleVersion: '2.0.0'", " dependencies:", " - uniqueId: 'uniqueid0'", " - version: '2.0.0'" @@ -184,22 +173,10 @@ public void setup() throws Exception { extensionNode = new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), Collections.emptyList() ); client = new NoOpNodeClient(this.getTestName()); @@ -220,63 +197,48 @@ public void testDiscover() throws Exception { ExtensionsManager extensionsManager = new ExtensionsManager(settings, extensionDir); - List expectedUninitializedExtensions = new ArrayList(); + List expectedExtensions = new ArrayList(); String expectedUniqueId = "uniqueid0"; Version expectedVersion = Version.fromString("2.0.0"); ExtensionDependency expectedDependency = new ExtensionDependency(expectedUniqueId, expectedVersion); - expectedUninitializedExtensions.add( + expectedExtensions.add( new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), Collections.emptyList() ) ); - expectedUninitializedExtensions.add( + expectedExtensions.add( new DiscoveryExtensionNode( "secondExtension", "uniqueid2", - "uniqueid2", - "myIndependentPluginHost2", - "127.0.0.1", - new TransportAddress(TransportAddress.META_ADDRESS, 9301), + new TransportAddress(InetAddress.getByName("127.0.0.1"), 9301), new HashMap(), Version.fromString("2.0.0"), - new PluginInfo( - "secondExtension", - "Fake description 2", - "3.14.16", - Version.fromString("2.0.0"), - "17", - "fakeClass2", - new ArrayList(), - true - ), + Version.fromString("2.0.0"), List.of(expectedDependency) ) ); - assertEquals(expectedUninitializedExtensions.size(), extensionsManager.getExtensionIdMap().values().size()); - assertEquals(List.of(expectedDependency), expectedUninitializedExtensions.get(1).getDependencies()); - assertTrue(expectedUninitializedExtensions.containsAll(extensionsManager.getExtensionIdMap().values())); - assertTrue(extensionsManager.getExtensionIdMap().values().containsAll(expectedUninitializedExtensions)); + assertEquals(expectedExtensions.size(), extensionsManager.getExtensionIdMap().values().size()); + assertEquals(List.of(expectedDependency), expectedExtensions.get(1).getDependencies()); + for (DiscoveryExtensionNode extension : expectedExtensions) { + DiscoveryExtensionNode initializedExtension = extensionsManager.getExtensionIdMap().get(extension.getId()); + assertEquals(extension.getName(), initializedExtension.getName()); + assertEquals(extension.getId(), initializedExtension.getId()); + assertEquals(extension.getAddress(), initializedExtension.getAddress()); + assertEquals(extension.getAttributes(), initializedExtension.getAttributes()); + assertEquals(extension.getVersion(), initializedExtension.getVersion()); + assertEquals(extension.getMinimumCompatibleVersion(), initializedExtension.getMinimumCompatibleVersion()); + // TODO: Will fail due to bug : https://github.com/opensearch-project/OpenSearch/issues/6115 + // assertEquals(extension.getDependencies(), initializedExtension.getDependencies()); + } } public void testNonUniqueExtensionsDiscovery() throws Exception { @@ -288,35 +250,32 @@ public void testNonUniqueExtensionsDiscovery() throws Exception { ExtensionsManager extensionsManager = new ExtensionsManager(settings, emptyExtensionDir); - List expectedUninitializedExtensions = new ArrayList(); + List expectedExtensions = new ArrayList(); - expectedUninitializedExtensions.add( + expectedExtensions.add( new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), Collections.emptyList() ) ); - assertEquals(expectedUninitializedExtensions.size(), extensionsManager.getExtensionIdMap().values().size()); - assertTrue(expectedUninitializedExtensions.containsAll(extensionsManager.getExtensionIdMap().values())); - assertTrue(extensionsManager.getExtensionIdMap().values().containsAll(expectedUninitializedExtensions)); - assertTrue(expectedUninitializedExtensions.containsAll(emptyList())); + assertEquals(expectedExtensions.size(), extensionsManager.getExtensionIdMap().values().size()); + for (DiscoveryExtensionNode extension : expectedExtensions) { + DiscoveryExtensionNode initializedExtension = extensionsManager.getExtensionIdMap().get(extension.getId()); + assertEquals(extension.getName(), initializedExtension.getName()); + assertEquals(extension.getId(), initializedExtension.getId()); + assertEquals(extension.getAddress(), initializedExtension.getAddress()); + assertEquals(extension.getAttributes(), initializedExtension.getAttributes()); + assertEquals(extension.getVersion(), initializedExtension.getVersion()); + assertEquals(extension.getMinimumCompatibleVersion(), initializedExtension.getMinimumCompatibleVersion()); + } + assertTrue(expectedExtensions.containsAll(emptyList())); + // TODO: Will fail due to bug : https://github.com/opensearch-project/OpenSearch/issues/6115 + // assertEquals(extension.getDependencies(), initializedExtension.getDependencies()); } public void testDiscoveryExtension() throws Exception { @@ -327,22 +286,10 @@ public void testDiscoveryExtension() throws Exception { DiscoveryExtensionNode discoveryExtensionNode = new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), List.of(expectedDependency) ); @@ -397,7 +344,7 @@ public void testNoExtensionsFile() throws Exception { new MockLogAppender.SeenEventExpectation( "No Extensions File Present", "org.opensearch.extensions.ExtensionsManager", - Level.INFO, + Level.WARN, "Extensions.yml file is not present. No extensions will be loaded." ) ); @@ -581,22 +528,10 @@ public void testExtensionDependencyResponse() throws Exception { new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), List.of(expectedDependency) ) ); @@ -837,6 +772,37 @@ public void testOnIndexModule() throws Exception { } + public void testIncompatibleExtensionRegistration() throws IOException, IllegalAccessException { + + try (MockLogAppender mockLogAppender = MockLogAppender.createForLoggers(LogManager.getLogger(ExtensionsManager.class))) { + + mockLogAppender.addExpectation( + new MockLogAppender.SeenEventExpectation( + "Could not load extension with uniqueId", + "org.opensearch.extensions.ExtensionsManager", + Level.ERROR, + "Could not load extension with uniqueId uniqueid1 due to OpenSearchException[Extension minimumCompatibleVersion: 3.99.0 is greater than current" + ) + ); + + List incompatibleExtension = Arrays.asList( + "extensions:", + " - name: firstExtension", + " uniqueId: uniqueid1", + " hostAddress: '127.0.0.0'", + " port: '9300'", + " version: '0.0.7'", + " opensearchVersion: '3.0.0'", + " minimumCompatibleVersion: '3.99.0'" + ); + + Files.write(extensionDir.resolve("extensions.yml"), incompatibleExtension, StandardCharsets.UTF_8); + ExtensionsManager extensionsManager = new ExtensionsManager(settings, extensionDir); + assertEquals(0, extensionsManager.getExtensionIdMap().values().size()); + mockLogAppender.assertAllExpectationsMatched(); + } + } + private void initialize(ExtensionsManager extensionsManager) { transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java index 276e47d7f55a8..551fb8a9bb277 100644 --- a/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java +++ b/server/src/test/java/org/opensearch/extensions/action/ExtensionTransportActionsHandlerTests.java @@ -24,7 +24,6 @@ import org.opensearch.extensions.RegisterTransportActionsRequest; import org.opensearch.extensions.rest.RestSendToExtensionActionTests; import org.opensearch.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.plugins.PluginInfo; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.client.NoOpNodeClient; import org.opensearch.test.transport.MockTransportService; @@ -37,7 +36,6 @@ import java.net.InetAddress; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -85,22 +83,10 @@ public void setup() throws Exception { discoveryExtensionNode = new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), Collections.emptyList() ); client = new NoOpNodeClient(this.getTestName()); diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 97eeae8fb95af..3b7f8b4c52c05 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -30,7 +30,6 @@ import org.opensearch.common.util.PageCacheRecycler; import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.indices.breaker.NoneCircuitBreakerService; -import org.opensearch.plugins.PluginInfo; import org.opensearch.rest.RestHandler.Route; import org.opensearch.rest.RestRequest.Method; import org.opensearch.test.OpenSearchTestCase; @@ -78,22 +77,10 @@ public void setup() throws Exception { discoveryExtensionNode = new DiscoveryExtensionNode( "firstExtension", "uniqueid1", - "uniqueid1", - "myIndependentPluginHost1", - "127.0.0.0", new TransportAddress(InetAddress.getByName("127.0.0.0"), 9300), new HashMap(), Version.fromString("3.0.0"), - new PluginInfo( - "firstExtension", - "Fake description 1", - "0.0.7", - Version.fromString("3.0.0"), - "14", - "fakeClass1", - new ArrayList(), - false - ), + Version.fromString("3.0.0"), Collections.emptyList() ); } diff --git a/server/src/test/resources/config/extensions.yml b/server/src/test/resources/config/extensions.yml deleted file mode 100644 index e02a2913385d9..0000000000000 --- a/server/src/test/resources/config/extensions.yml +++ /dev/null @@ -1,18 +0,0 @@ -extensions: - - name: firstExtension - uniqueId: uniqueid1 - hostName: 'myIndependentPluginHost1' - hostAddress: '127.0.0.0' - port: '9300' - version: '3.0.0' - - name: "secondExtension" - uniqueId: 'uniqueid2' - dependencies: - - name: 'uniqueid0' - version: '2.0.0' - - name: 'uniqueid1' - version: '3.0.0' - hostName: 'myIndependentPluginHost2' - hostAddress: '127.0.0.1' - port: '9301' - version: '2.0.0' From 073b7366cd4d7367268b9ebdbaa37e2ba2439008 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 1 Feb 2023 15:00:55 -0500 Subject: [PATCH 4/9] Bumps Netty from 4.1.86.Final to 4.1.87.Final (#6130) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- .../licenses/netty-buffer-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-buffer-4.1.87.Final.jar.sha1 | 1 + .../transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1 | 1 - .../transport-netty4/licenses/netty-codec-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-common-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-common-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-handler-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-handler-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-resolver-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-resolver-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.87.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.86.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-dns-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-dns-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-http2-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-http2-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-socks-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-socks-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-handler-proxy-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-handler-proxy-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-resolver-dns-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-resolver-dns-4.1.87.Final.jar.sha1 | 1 + .../netty-transport-native-unix-common-4.1.86.Final.jar.sha1 | 1 - .../netty-transport-native-unix-common-4.1.87.Final.jar.sha1 | 1 + .../repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1 | 1 - .../repository-hdfs/licenses/netty-all-4.1.87.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-buffer-4.1.87.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-codec-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-codec-http-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-codec-http-4.1.87.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-common-4.1.87.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-handler-4.1.87.Final.jar.sha1 | 1 + .../transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1 | 1 - .../transport-nio/licenses/netty-resolver-4.1.87.Final.jar.sha1 | 1 + .../licenses/netty-transport-4.1.86.Final.jar.sha1 | 1 - .../licenses/netty-transport-4.1.87.Final.jar.sha1 | 1 + 48 files changed, 25 insertions(+), 24 deletions(-) delete mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-buffer-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-common-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-handler-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-resolver-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-4.1.87.Final.jar.sha1 delete mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 create mode 100644 modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-dns-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-codec-socks-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-handler-proxy-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-resolver-dns-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/netty-all-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-buffer-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-codec-http-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-common-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-handler-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-resolver-4.1.87.Final.jar.sha1 delete mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.86.Final.jar.sha1 create mode 100644 plugins/transport-nio/licenses/netty-transport-4.1.87.Final.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 272f4deefd713..26128c1c38228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947)) - Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008)) - [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976)) +- Bumps `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130)) ### Deprecated diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 2599c38dbcb2f..33bfff73b9512 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -25,7 +25,7 @@ antlr4 = 4.11.1 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.5.0 -netty = 4.1.86.Final +netty = 4.1.87.Final joda = 2.12.2 # client dependencies diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1 deleted file mode 100644 index c477a0d3b0ee9..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a66fa0ed2687eb33a2e53a17a6df61bfe3b3f2bd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..8350f48acdb54 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +8942174141fb0586e7fffefc8ae48e1458293696 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1 deleted file mode 100644 index b2bd305825d88..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee126da926ea202da3b21eb737788ef83b1db772 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..9a494c68b7afb --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +d7ab7363f736114e6324ccf802d094529b30b8d8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1 deleted file mode 100644 index 60affc4a1faed..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -23674593f004959ae002ec348626eecf677191ae \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..115657578cf80 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +3230e95d6832ee4306fe1ca774c7bcecf1da4b28 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 deleted file mode 100644 index e0fb5c637d571..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ac0ca067e4118533ad1038776fcd9d5f3058b7d4 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..da15720c2d0e1 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +72b756ff290d782c9ebb36db7d42ee272270c0b4 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1 deleted file mode 100644 index 48c07b3c9f5df..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1dceab4662a9cc93faf87b237bb41103b1bc7f0e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..066550b9e0135 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +3e7e80dd6e604144781fcb859b79cfe8d3730079 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1 deleted file mode 100644 index 7c036b195f091..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0bcb65230218286e6456b5d085cb42e67776eb70 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..0923602100814 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +2bd97491c22ebea4670c00f1bd5dbf65a8a1cfe7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1 deleted file mode 100644 index f5258c46ebd6a..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bad83d479f7bd8ea84eefd77c316435be4c97270 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..4465a47bd49fb --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +eaa964e16a67914c8d9b814d29a4b969635d72a0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1 deleted file mode 100644 index 1fa4ab0281ca1..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7c7739c41fd110c3576e9faace332ee957f27203 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..dcb49c515e460 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +30c78c8ced3417f35e2a55f7533dc2bb43fef2aa \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 deleted file mode 100644 index 3701a94dc9aec..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e0e7fc1c337485cabcf7971faefe692b76f93a2 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..4fb195e5f08f6 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +4bf8308dc7d6ba2570681eaeddb16f8d1196b438 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.86.Final.jar.sha1 deleted file mode 100644 index 9a8ebe2fb1be3..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c8de479f36a8457541fcbb0016c851bde3e67693 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..ea9c36f36b988 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +092ec0019d4e72c1305f4357694c4831aade60ce \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 deleted file mode 100644 index e0fb5c637d571..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ac0ca067e4118533ad1038776fcd9d5f3058b7d4 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..da15720c2d0e1 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +72b756ff290d782c9ebb36db7d42ee272270c0b4 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.86.Final.jar.sha1 deleted file mode 100644 index 6544ba9942c96..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -854264e7ad75887bc25b82eb38e4ee65c8b44dc3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..5bd83f816a9e4 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +109027470b72d56914d76154e947f9bd5844bb9b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.86.Final.jar.sha1 deleted file mode 100644 index 5f8a3056159f5..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2515d76be9671cc248bab77352edddd16bfa9436 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..226ea309ac472 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +8079f16d6209e10f5d3a09522e7aa9ffb217e17e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.86.Final.jar.sha1 deleted file mode 100644 index b73e612b2a8c6..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a1e2ef79e4944b5d38092328c36c68e677a4b5f3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..39d59cb45eed2 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +a40442ff2afaf73d7ddc1d75d7e86814ef04f980 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 deleted file mode 100644 index 3701a94dc9aec..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5e0e7fc1c337485cabcf7971faefe692b76f93a2 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..4fb195e5f08f6 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +4bf8308dc7d6ba2570681eaeddb16f8d1196b438 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1 deleted file mode 100644 index 75cb32ca4b323..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a6395c3d2f8699e8dc4fd1e38171f82045f4af7b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.87.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..8dcb64ecfdf24 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +39de92ea74a05da937343695f780e59addb8b8ea \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1 deleted file mode 100644 index c477a0d3b0ee9..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a66fa0ed2687eb33a2e53a17a6df61bfe3b3f2bd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..8350f48acdb54 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +8942174141fb0586e7fffefc8ae48e1458293696 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1 deleted file mode 100644 index b2bd305825d88..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee126da926ea202da3b21eb737788ef83b1db772 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..9a494c68b7afb --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +d7ab7363f736114e6324ccf802d094529b30b8d8 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.86.Final.jar.sha1 deleted file mode 100644 index 60affc4a1faed..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -23674593f004959ae002ec348626eecf677191ae \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..115657578cf80 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +3230e95d6832ee4306fe1ca774c7bcecf1da4b28 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1 deleted file mode 100644 index 48c07b3c9f5df..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1dceab4662a9cc93faf87b237bb41103b1bc7f0e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..066550b9e0135 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +3e7e80dd6e604144781fcb859b79cfe8d3730079 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1 deleted file mode 100644 index 7c036b195f091..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0bcb65230218286e6456b5d085cb42e67776eb70 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..0923602100814 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +2bd97491c22ebea4670c00f1bd5dbf65a8a1cfe7 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1 deleted file mode 100644 index f5258c46ebd6a..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bad83d479f7bd8ea84eefd77c316435be4c97270 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..4465a47bd49fb --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +eaa964e16a67914c8d9b814d29a4b969635d72a0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.86.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.86.Final.jar.sha1 deleted file mode 100644 index 1fa4ab0281ca1..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.86.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7c7739c41fd110c3576e9faace332ee957f27203 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.87.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.87.Final.jar.sha1 new file mode 100644 index 0000000000000..dcb49c515e460 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.87.Final.jar.sha1 @@ -0,0 +1 @@ +30c78c8ced3417f35e2a55f7533dc2bb43fef2aa \ No newline at end of file From 73c51c70ee36863c487b356214cd1769ce9c97b2 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 1 Feb 2023 12:05:09 -0800 Subject: [PATCH 5/9] Add release notes for 1.3.8 (#6134) Signed-off-by: Andrew Ross --- release-notes/opensearch.release-notes-1.3.8.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 release-notes/opensearch.release-notes-1.3.8.md diff --git a/release-notes/opensearch.release-notes-1.3.8.md b/release-notes/opensearch.release-notes-1.3.8.md new file mode 100644 index 0000000000000..f87558cc09da6 --- /dev/null +++ b/release-notes/opensearch.release-notes-1.3.8.md @@ -0,0 +1,13 @@ +## 2023-02-01 Version 1.3.8 Release Notes + +### Upgrades +* Upgrade Netty to 4.1.86.Final ([#5548](https://github.com/opensearch-project/OpenSearch/pull/5548)) +* Upgrade zookeeper dependency in hdfs-fixture ([#5007](https://github.com/opensearch-project/OpenSearch/pull/5007)) +* Upgrade `woodstox-core` to 6.4.0 ([#4947](https://github.com/opensearch-project/OpenSearch/pull/4947)) +* Upgrade `tika` from 2.4.0 to 2.5.0 ([#4791](https://github.com/opensearch-project/OpenSearch/pull/4791)) +* Upgrade `jettison` from 1.5.0 to 1.5.3 +* Upgrade `protobuf-java` to 3.21.9 ([#5319](https://github.com/opensearch-project/OpenSearch/pull/5319)) +* Upgrade hadoop-minicluster to 3.3.4 ([#6034](https://github.com/opensearch-project/OpenSearch/pull/6034)) +* Upgrade aws-java-sdk to 1.12.270 ([#6035](https://github.com/opensearch-project/OpenSearch/pull/6035)) +* Upgrade reactor-netty-http to 1.0.24 in repository-azure ([#4880](https://github.com/opensearch-project/OpenSearch/pull/4880)) +* Upgrade jetty-http, kotlin-stdlib and snakeyaml ([#4981](https://github.com/opensearch-project/OpenSearch/pull/4981)) From da1133762a2ae0f81b7d0a082ae479759f6dacdc Mon Sep 17 00:00:00 2001 From: Bryan Burkholder <771133+bryanlb@users.noreply.github.com> Date: Wed, 1 Feb 2023 13:40:21 -0700 Subject: [PATCH 6/9] Fix incorrect exception message for ValuesSourceConfig (#6131) Signed-off-by: Bryan Burkholder --- .../search/aggregations/support/ValuesSourceConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceConfig.java index e956dd054eba9..b5bd355bb5847 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceConfig.java @@ -323,7 +323,7 @@ public ValuesSourceConfig( if (!valid()) { // TODO: resolve no longer generates invalid configs. Once VSConfig is immutable, we can drop this check throw new IllegalStateException( - "value source config is invalid; must have either a field context or a script or marked as unwrapped" + "value source config is invalid; must have either a field context or a script or marked as unmapped" ); } valuesSource = ConstructValuesSource(missing, format, nowSupplier); From dab77e61f445346614828bd6643d97012c7f51f6 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 1 Feb 2023 16:03:57 -0500 Subject: [PATCH 7/9] Bumps Jackson from 2.4.1 to 2.4.2 (#6129) Signed-off-by: Andriy Redko --- CHANGELOG.md | 3 ++- buildSrc/version.properties | 4 ++-- client/sniffer/licenses/jackson-core-2.14.1.jar.sha1 | 1 - client/sniffer/licenses/jackson-core-2.14.2.jar.sha1 | 1 + .../upgrade-cli/licenses/jackson-annotations-2.14.1.jar.sha1 | 1 - .../upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1 | 1 + .../upgrade-cli/licenses/jackson-databind-2.14.1.jar.sha1 | 1 - .../upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1 | 1 + libs/x-content/licenses/jackson-core-2.14.1.jar.sha1 | 1 - libs/x-content/licenses/jackson-core-2.14.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-cbor-2.14.1.jar.sha1 | 1 - .../licenses/jackson-dataformat-cbor-2.14.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-smile-2.14.1.jar.sha1 | 1 - .../licenses/jackson-dataformat-smile-2.14.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-yaml-2.14.1.jar.sha1 | 1 - .../licenses/jackson-dataformat-yaml-2.14.2.jar.sha1 | 1 + .../ingest-geoip/licenses/jackson-annotations-2.14.1.jar.sha1 | 1 - .../ingest-geoip/licenses/jackson-annotations-2.14.2.jar.sha1 | 1 + .../ingest-geoip/licenses/jackson-databind-2.14.1.jar.sha1 | 1 - .../ingest-geoip/licenses/jackson-databind-2.14.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.14.1.jar.sha1 | 1 - .../licenses/jackson-annotations-2.14.2.jar.sha1 | 1 + .../discovery-ec2/licenses/jackson-databind-2.14.1.jar.sha1 | 1 - .../discovery-ec2/licenses/jackson-databind-2.14.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.14.1.jar.sha1 | 1 - .../licenses/jackson-annotations-2.14.2.jar.sha1 | 1 + .../licenses/jackson-databind-2.14.1.jar.sha1 | 1 - .../licenses/jackson-databind-2.14.2.jar.sha1 | 1 + .../licenses/jackson-dataformat-xml-2.14.1.jar.sha1 | 1 - .../licenses/jackson-dataformat-xml-2.14.2.jar.sha1 | 1 + .../licenses/jackson-datatype-jsr310-2.14.1.jar.sha1 | 1 - .../licenses/jackson-datatype-jsr310-2.14.2.jar.sha1 | 1 + .../licenses/jackson-module-jaxb-annotations-2.14.1.jar.sha1 | 1 - .../licenses/jackson-module-jaxb-annotations-2.14.2.jar.sha1 | 1 + .../licenses/jackson-annotations-2.14.1.jar.sha1 | 1 - .../licenses/jackson-annotations-2.14.2.jar.sha1 | 1 + .../repository-s3/licenses/jackson-databind-2.14.1.jar.sha1 | 1 - .../repository-s3/licenses/jackson-databind-2.14.2.jar.sha1 | 1 + 38 files changed, 22 insertions(+), 21 deletions(-) delete mode 100644 client/sniffer/licenses/jackson-core-2.14.1.jar.sha1 create mode 100644 client/sniffer/licenses/jackson-core-2.14.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.1.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.1.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-core-2.14.1.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-core-2.14.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.14.1.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.14.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.14.1.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.14.2.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.14.1.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.14.2.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.14.1.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.14.2.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.14.1.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.14.2.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.14.1.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.14.2.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.14.1.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.14.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.14.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.14.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-databind-2.14.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-databind-2.14.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.2.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.2.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.14.1.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.14.2.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.14.1.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.14.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 26128c1c38228..5d09520fac6d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,13 +91,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bumps `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6076](https://github.com/opensearch-project/OpenSearch/pull/6076)) - Bumps `joda` from 2.10.13 to 2.12.2 ([#6083](https://github.com/opensearch-project/OpenSearch/pull/6083)) - Upgrade to Lucene 9.5.0 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/6078)) +- Bumps `Jackson` from 2.14.1 to 2.14.2 ([#6129](https://github.com/opensearch-project/OpenSearch/pull/6129)) +- Bumps `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130)) ### Changed - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725)) - [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947)) - Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008)) - [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976)) -- Bumps `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130)) ### Deprecated diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 33bfff73b9512..c7fd0fc0abf09 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -8,8 +8,8 @@ bundled_jdk = 19.0.2+7 # optional dependencies spatial4j = 0.7 jts = 1.15.0 -jackson = 2.14.1 -jackson_databind = 2.14.1 +jackson = 2.14.2 +jackson_databind = 2.14.2 snakeyaml = 1.32 icu4j = 70.1 supercsv = 2.4.0 diff --git a/client/sniffer/licenses/jackson-core-2.14.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.14.1.jar.sha1 deleted file mode 100644 index 054873b60eb21..0000000000000 --- a/client/sniffer/licenses/jackson-core-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a07bc535ccf0b7f6929c4d0f2ab9b294ef7c4a3 \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-2.14.2.jar.sha1 b/client/sniffer/licenses/jackson-core-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..5a241779e03c9 --- /dev/null +++ b/client/sniffer/licenses/jackson-core-2.14.2.jar.sha1 @@ -0,0 +1 @@ +f804090e6399ce0cf78242db086017512dd71fcc \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.1.jar.sha1 deleted file mode 100644 index e43faef9e23ff..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a6ad504d591a7903ffdec76b5b7252819a2d162 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..bfa80ea001fcf --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +a7aae9525864930723e3453ab799521fdfd9d873 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.1.jar.sha1 deleted file mode 100644 index 0e6726927ebac..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -268524b9056cae1211b9f1f52560ef19347f4d17 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..1e2ec2f7f6587 --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1 @@ -0,0 +1 @@ +01e71fddbc80bb86f71a6345ac1e8ab8a00e7134 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.14.1.jar.sha1 b/libs/x-content/licenses/jackson-core-2.14.1.jar.sha1 deleted file mode 100644 index 054873b60eb21..0000000000000 --- a/libs/x-content/licenses/jackson-core-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a07bc535ccf0b7f6929c4d0f2ab9b294ef7c4a3 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.14.2.jar.sha1 b/libs/x-content/licenses/jackson-core-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..5a241779e03c9 --- /dev/null +++ b/libs/x-content/licenses/jackson-core-2.14.2.jar.sha1 @@ -0,0 +1 @@ +f804090e6399ce0cf78242db086017512dd71fcc \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.14.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.14.1.jar.sha1 deleted file mode 100644 index e1dcda6b33782..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04e6fbcdcd2a01e4a5cb5901338cab6199c9b26b \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.14.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..45de2f3a64e95 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.14.2.jar.sha1 @@ -0,0 +1 @@ +5999f5365e27b63b27a52e671d15c0f8f8346873 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.14.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.14.1.jar.sha1 deleted file mode 100644 index 7138ebda0e78c..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -656ccecc1fc85b95d13e5b8080289fc1a5e5e21e \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.14.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..6e482752909ee --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-2.14.2.jar.sha1 @@ -0,0 +1 @@ +d03e3b991a1cb96357b98d07fcbe42d3d5c0b496 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.14.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.14.1.jar.sha1 deleted file mode 100644 index 300b6920dfc8d..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cf6d18651659a2e64301452c841e6daa62e77bf6 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.14.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..e1cf562bbcc3f --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.14.2.jar.sha1 @@ -0,0 +1 @@ +cc9a25c1f4212562dcb2fa33dd8ae179ba0e6a4e \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.14.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.14.1.jar.sha1 deleted file mode 100644 index e43faef9e23ff..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a6ad504d591a7903ffdec76b5b7252819a2d162 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.14.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..bfa80ea001fcf --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +a7aae9525864930723e3453ab799521fdfd9d873 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.14.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.14.1.jar.sha1 deleted file mode 100644 index 0e6726927ebac..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -268524b9056cae1211b9f1f52560ef19347f4d17 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.14.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..1e2ec2f7f6587 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.14.2.jar.sha1 @@ -0,0 +1 @@ +01e71fddbc80bb86f71a6345ac1e8ab8a00e7134 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.14.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.14.1.jar.sha1 deleted file mode 100644 index e43faef9e23ff..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a6ad504d591a7903ffdec76b5b7252819a2d162 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.14.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..bfa80ea001fcf --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +a7aae9525864930723e3453ab799521fdfd9d873 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.14.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.14.1.jar.sha1 deleted file mode 100644 index 0e6726927ebac..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -268524b9056cae1211b9f1f52560ef19347f4d17 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.14.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..1e2ec2f7f6587 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.14.2.jar.sha1 @@ -0,0 +1 @@ +01e71fddbc80bb86f71a6345ac1e8ab8a00e7134 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.14.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.14.1.jar.sha1 deleted file mode 100644 index e43faef9e23ff..0000000000000 --- a/plugins/repository-azure/licenses/jackson-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a6ad504d591a7903ffdec76b5b7252819a2d162 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.14.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..bfa80ea001fcf --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +a7aae9525864930723e3453ab799521fdfd9d873 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.14.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.14.1.jar.sha1 deleted file mode 100644 index 0e6726927ebac..0000000000000 --- a/plugins/repository-azure/licenses/jackson-databind-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -268524b9056cae1211b9f1f52560ef19347f4d17 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.14.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..1e2ec2f7f6587 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-databind-2.14.2.jar.sha1 @@ -0,0 +1 @@ +01e71fddbc80bb86f71a6345ac1e8ab8a00e7134 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.1.jar.sha1 deleted file mode 100644 index d4b883fb92650..0000000000000 --- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ccd98bd674080338a6ca4bcdd52be7fb465cec1d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..41d05aa71a3a0 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.14.2.jar.sha1 @@ -0,0 +1 @@ +1c162dd3006c0b7608becd55af4639f2c7b5a79a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.1.jar.sha1 deleted file mode 100644 index 4eac9019ac93c..0000000000000 --- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f24e8cb1437e05149b7a3049ebd6700f42e664b1 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..d4d8a1b82ebc2 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.14.2.jar.sha1 @@ -0,0 +1 @@ +796518148a385b2728d44886cc0f8852eb8eeb53 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.1.jar.sha1 deleted file mode 100644 index a3f1ff40d44f1..0000000000000 --- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c986d9cc542fe5ade8aaebf5f0360a563dc51762 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..cf26b10e6b60a --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +f7a5457c02d83103710973a4ffdce430ccdc1fd2 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.14.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.14.1.jar.sha1 deleted file mode 100644 index e43faef9e23ff..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a6ad504d591a7903ffdec76b5b7252819a2d162 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.14.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..bfa80ea001fcf --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.14.2.jar.sha1 @@ -0,0 +1 @@ +a7aae9525864930723e3453ab799521fdfd9d873 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.14.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.14.1.jar.sha1 deleted file mode 100644 index 0e6726927ebac..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.14.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -268524b9056cae1211b9f1f52560ef19347f4d17 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.14.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.14.2.jar.sha1 new file mode 100644 index 0000000000000..1e2ec2f7f6587 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.14.2.jar.sha1 @@ -0,0 +1 @@ +01e71fddbc80bb86f71a6345ac1e8ab8a00e7134 \ No newline at end of file From a7d38845b1b2fed2fa2bde28e397a85cbf337eac Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 1 Feb 2023 14:40:26 -0700 Subject: [PATCH 8/9] Fix flaky test IndexStatsIT.testFilterCacheStats. (#5963) This change disables query cache when internally querying for operation uid. This code path is executed only when assertions are enabled to check if an op already exists in the index. The current code fetches the searcher using acquireSearcher, which reuses the queryCache and conflicts with tests on the cache. Signed-off-by: Marc Handalian --- .../main/java/org/opensearch/index/engine/InternalEngine.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index f50b5f0ca8c74..55aad817a5816 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -1289,6 +1289,7 @@ private boolean assertDocDoesNotExist(final Index index, final boolean allowDele } } else { try (Searcher searcher = acquireSearcher("assert doc doesn't exist", SearcherScope.INTERNAL)) { + searcher.setQueryCache(null); final long docsWithId = searcher.count(new TermQuery(index.uid())); if (docsWithId > 0) { throw new AssertionError("doc [" + index.id() + "] exists [" + docsWithId + "] times in index"); From 53d54de90423e31af7ca2514b953d77df4ac2be4 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 1 Feb 2023 15:52:53 -0800 Subject: [PATCH 9/9] [Segment Replication] For replica recovery, force segment replication sync from peer recovery source (#5746) * [Segment Replication] For replica recovery, force segment replication sync from source Signed-off-by: Suraj Singh * Rebase against main Signed-off-by: Suraj Singh * Fix unit test Signed-off-by: Suraj Singh * PR feedback Signed-off-by: Suraj Singh * Fix remote store recovery test Signed-off-by: Suraj Singh --------- Signed-off-by: Suraj Singh --- .../replication/SegmentReplicationIT.java | 79 ----------- .../SegmentReplicationRelocationIT.java | 129 ++++++++++++++++++ .../cluster/IndicesClusterStateService.java | 77 +---------- .../recovery/RecoverySourceHandler.java | 5 + .../SegmentReplicationTargetService.java | 7 +- .../SegmentReplicationIndexShardTests.java | 21 +-- ...teStorePeerRecoverySourceHandlerTests.java | 8 +- .../SegmentReplicationSourceHandlerTests.java | 3 +- .../SegmentReplicationTargetServiceTests.java | 4 +- ...enSearchIndexLevelReplicationTestCase.java | 2 +- .../index/shard/IndexShardTestCase.java | 14 +- 11 files changed, 170 insertions(+), 179 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 0101379321932..dfe27c54444d0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -9,20 +9,15 @@ package org.opensearch.indices.replication; import com.carrotsearch.randomizedtesting.RandomizedTest; -import org.opensearch.OpenSearchCorruptionException; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Requests; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; -import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexModule; import org.opensearch.index.shard.IndexShard; -import org.opensearch.indices.IndicesService; import org.opensearch.indices.recovery.FileChunkRequest; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.BackgroundIndexer; @@ -36,7 +31,6 @@ import java.util.concurrent.CountDownLatch; import static java.util.Arrays.asList; -import static org.hamcrest.Matchers.equalTo; import static org.opensearch.index.query.QueryBuilders.matchQuery; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -139,79 +133,6 @@ public void testCancelPrimaryAllocation() throws Exception { verifyStoreContent(); } - /** - * This test verfies that replica shard is not added to the cluster when doing a round of segment replication fails during peer recovery. - *

- * TODO: Ignoring this test as its flaky and needs separate fix - */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/5669") - public void testAddNewReplicaFailure() throws Exception { - logger.info("--> starting [Primary Node] ..."); - final String primaryNode = internalCluster().startNode(); - - logger.info("--> creating test index ..."); - prepareCreate( - INDEX_NAME, - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - - ).get(); - - logger.info("--> index 10 docs"); - for (int i = 0; i < 10; i++) { - client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); - } - logger.info("--> flush so we have some segment files on disk"); - flush(INDEX_NAME); - logger.info("--> index more docs so we have something in the translog"); - for (int i = 10; i < 20; i++) { - client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); - } - refresh(INDEX_NAME); - logger.info("--> verifying count"); - assertThat(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(20L)); - - logger.info("--> start empty node to add replica shard"); - final String replicaNode = internalCluster().startNode(); - - // Mock transport service to add behaviour of throwing corruption exception during segment replication process. - MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance( - TransportService.class, - primaryNode - )); - mockTransportService.addSendBehavior( - internalCluster().getInstance(TransportService.class, replicaNode), - (connection, requestId, action, request, options) -> { - if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) { - throw new OpenSearchCorruptionException("expected"); - } - connection.sendRequest(requestId, action, request, options); - } - ); - ensureGreen(INDEX_NAME); - // Add Replica shard to the new empty replica node - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings(INDEX_NAME) - .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) - ); - - // Verify that cluster state is not green and replica shard failed during a round of segment replication is not added to the cluster - ClusterHealthResponse clusterHealthResponse = client().admin() - .cluster() - .prepareHealth() - .setWaitForEvents(Priority.LANGUID) - .setWaitForNodes("2") - .setWaitForGreenStatus() - .setTimeout(TimeValue.timeValueSeconds(2)) - .execute() - .actionGet(); - assertTrue(clusterHealthResponse.isTimedOut()); - ensureYellow(INDEX_NAME); - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, replicaNode); - assertFalse(indicesService.hasIndex(resolveIndex(INDEX_NAME))); - } - public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeB = internalCluster().startNode(); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java index 681150fb2bd83..4f0983553de7c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationRelocationIT.java @@ -22,6 +22,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexModule; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -32,6 +33,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + /** * This test class verifies primary shard relocation with segment replication as replication strategy. */ @@ -398,4 +401,130 @@ public void testRelocateWithQueuedOperationsDuringHandoff() throws Exception { waitForSearchableDocs(totalDocCount, replica, newPrimary); verifyStoreContent(); } + + /** + * This test verifies that adding a new node which results in peer recovery as replica; also bring replica's + * replication checkpoint upto the primary's by performing a round of segment replication. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/5669") + public void testNewlyAddedReplicaIsUpdated() throws Exception { + final String primary = internalCluster().startNode(); + prepareCreate( + INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ).get(); + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + logger.info("--> flush so we have some segment files on disk"); + flush(INDEX_NAME); + logger.info("--> index more docs so we have something in the translog"); + for (int i = 10; i < 20; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + refresh(INDEX_NAME); + assertEquals(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, 20L); + + logger.info("--> start empty node to add replica shard"); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + // Update replica count settings to 1 so that peer recovery triggers and recover replica + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + + ClusterHealthResponse clusterHealthResponse = client().admin() + .cluster() + .prepareHealth() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNodes("2") + .setWaitForGreenStatus() + .setTimeout(TimeValue.timeValueSeconds(2)) + .execute() + .actionGet(); + assertFalse(clusterHealthResponse.isTimedOut()); + ensureGreen(INDEX_NAME); + flushAndRefresh(INDEX_NAME); + waitForSearchableDocs(20, primary, replica); + verifyStoreContent(); + } + + /** + * This test verifies that replica shard is not added to the cluster when doing a round of segment replication fails during peer recovery. + * + * TODO: Ignoring this test as its flaky and needs separate fix + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/5669") + public void testAddNewReplicaFailure() throws Exception { + logger.info("--> starting [Primary Node] ..."); + final String primaryNode = internalCluster().startNode(); + + logger.info("--> creating test index ..."); + prepareCreate( + INDEX_NAME, + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + + ).get(); + + logger.info("--> index 10 docs"); + for (int i = 0; i < 10; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + logger.info("--> flush so we have some segment files on disk"); + flush(INDEX_NAME); + logger.info("--> index more docs so we have something in the translog"); + for (int i = 10; i < 20; i++) { + client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + refresh(INDEX_NAME); + logger.info("--> verifying count"); + assertEquals(client().prepareSearch(INDEX_NAME).setSize(0).execute().actionGet().getHits().getTotalHits().value, 20L); + + logger.info("--> start empty node to add replica shard"); + final String replica = internalCluster().startNode(); + + final CountDownLatch waitForRecovery = new CountDownLatch(1); + // Mock transport service to add behaviour of throwing corruption exception during segment replication process. + MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance( + TransportService.class, + primaryNode + )); + mockTransportService.addSendBehavior( + internalCluster().getInstance(TransportService.class, replica), + (connection, requestId, action, request, options) -> { + if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) { + waitForRecovery.countDown(); + throw new OpenSearchCorruptionException("expected"); + } + connection.sendRequest(requestId, action, request, options); + } + ); + ensureGreen(INDEX_NAME); + // Add Replica shard to the new empty replica node + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(INDEX_NAME) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, replica); + waitForRecovery.await(); + assertBusy(() -> assertTrue(indicesService.hasIndex(resolveIndex(INDEX_NAME)))); + + // Verify that cluster state is not green and replica shard failed during a round of segment replication is not added to the cluster + ClusterHealthResponse clusterHealthResponse = client().admin() + .cluster() + .prepareHealth() + .setWaitForEvents(Priority.LANGUID) + .setWaitForNodes("2") + .setWaitForGreenStatus() + .setTimeout(TimeValue.timeValueSeconds(2)) + .execute() + .actionGet(); + assertTrue(clusterHealthResponse.isTimedOut()); + ensureYellow(INDEX_NAME); + } } diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java index e8adcbdc1c89a..966e2168e263c 100644 --- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java +++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; -import org.opensearch.action.StepListener; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateApplier; @@ -83,11 +82,8 @@ import org.opensearch.indices.recovery.RecoveryListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.indices.replication.SegmentReplicationSourceService; -import org.opensearch.indices.replication.SegmentReplicationState; import org.opensearch.indices.replication.SegmentReplicationTargetService; -import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; -import org.opensearch.indices.replication.common.ReplicationFailedException; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.repositories.RepositoriesService; import org.opensearch.search.SearchService; @@ -781,78 +777,7 @@ public synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolea public void handleRecoveryDone(ReplicationState state, ShardRouting shardRouting, long primaryTerm) { RecoveryState recoveryState = (RecoveryState) state; - AllocatedIndex indexService = indicesService.indexService(shardRouting.shardId().getIndex()); - StepListener forceSegRepListener = new StepListener<>(); - // For Segment Replication enabled indices, we want replica shards to start a replication event to fetch latest segments before - // it is marked as Started. - if (indexService.getIndexSettings().isSegRepEnabled()) { - forceSegmentReplication(indexService, shardRouting, forceSegRepListener); - } else { - forceSegRepListener.onResponse(null); - } - forceSegRepListener.whenComplete( - v -> shardStateAction.shardStarted( - shardRouting, - primaryTerm, - "after " + recoveryState.getRecoverySource(), - SHARD_STATE_ACTION_LISTENER - ), - e -> handleRecoveryFailure(shardRouting, true, e) - ); - } - - /** - * Forces a round of Segment Replication with empty checkpoint, so that replicas could fetch latest segment files from primary. - */ - private void forceSegmentReplication( - AllocatedIndex indexService, - ShardRouting shardRouting, - StepListener forceSegRepListener - ) { - IndexShard indexShard = (IndexShard) indexService.getShardOrNull(shardRouting.id()); - if (indexShard != null && indexShard.isSegmentReplicationAllowed()) { - segmentReplicationTargetService.startReplication( - ReplicationCheckpoint.empty(shardRouting.shardId()), - indexShard, - new SegmentReplicationTargetService.SegmentReplicationListener() { - @Override - public void onReplicationDone(SegmentReplicationState state) { - logger.trace( - () -> new ParameterizedMessage( - "[shardId {}] [replication id {}] Replication complete, timing data: {}", - indexShard.shardId().getId(), - state.getReplicationId(), - state.getTimingData() - ) - ); - forceSegRepListener.onResponse(null); - } - - @Override - public void onReplicationFailure( - SegmentReplicationState state, - ReplicationFailedException e, - boolean sendShardFailure - ) { - logger.trace( - () -> new ParameterizedMessage( - "[shardId {}] [replication id {}] Replication failed, timing data: {}", - indexShard.shardId().getId(), - state.getReplicationId(), - state.getTimingData() - ) - ); - if (sendShardFailure == true) { - logger.error("replication failure", e); - indexShard.failShard("replication failure", e); - } - forceSegRepListener.onFailure(e); - } - } - ); - } else { - forceSegRepListener.onResponse(null); - } + shardStateAction.shardStarted(shardRouting, primaryTerm, "after " + recoveryState.getRecoverySource(), SHARD_STATE_ACTION_LISTENER); } private void failAndRemoveShard( diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java index 4f3e68aba16f3..212b5c8bc3d96 100644 --- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandler.java @@ -832,6 +832,11 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis * if the recovery process fails after disabling primary mode on the source shard, both relocation source and * target are failed (see {@link IndexShard#updateRoutingEntry}). */ + } else { + // Force round of segment replication to update its checkpoint to primary's + if (shard.indexSettings().isSegRepEnabled()) { + recoveryTarget.forceSegmentFileSync(); + } } stopWatch.stop(); logger.info("finalizing recovery took [{}]", stopWatch.totalTime()); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index 4a20e4994cbee..3e1902bbe8ec5 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -347,7 +347,7 @@ public void messageReceived(final FileChunkRequest request, TransportChannel cha } } - class ForceSyncTransportRequestHandler implements TransportRequestHandler { + private class ForceSyncTransportRequestHandler implements TransportRequestHandler { @Override public void messageReceived(final ForceSyncRequest request, TransportChannel channel, Task task) throws Exception { assert indicesService != null; @@ -367,7 +367,10 @@ public void onReplicationDone(SegmentReplicationState state) { ) ); try { - indexShard.resetToWriteableEngine(); + // Promote engine type for primary target + if (indexShard.recoveryState().getPrimary() == true) { + indexShard.resetToWriteableEngine(); + } channel.sendResponse(TransportResponse.Empty.INSTANCE); } catch (InterruptedException | TimeoutException | IOException e) { throw new RuntimeException(e); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 54e8682f90d22..31cf2f84ddf01 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -518,18 +518,17 @@ public void testReplicaReceivesLowerGeneration() throws Exception { replicateSegments(primary, List.of(replica_1)); assertEqualCommittedSegments(primary, replica_1); - assertLatestCommitGen(4, primary, replica_1); - assertLatestCommitGen(2, replica_2); + assertLatestCommitGen(4, primary); + assertLatestCommitGen(5, replica_1); + assertLatestCommitGen(3, replica_2); shards.promoteReplicaToPrimary(replica_2).get(); primary.close("demoted", false); primary.store().close(); IndexShard oldPrimary = shards.addReplicaWithExistingPath(primary.shardPath(), primary.routingEntry().currentNodeId()); shards.recoverReplica(oldPrimary); - assertLatestCommitGen(4, oldPrimary); - assertEqualCommittedSegments(oldPrimary, replica_1); - - assertLatestCommitGen(4, replica_2); + assertLatestCommitGen(5, oldPrimary); + assertLatestCommitGen(5, replica_2); numDocs = randomIntBetween(numDocs + 1, numDocs + 10); shards.indexDocs(numDocs); @@ -716,10 +715,12 @@ public void testNRTReplicaPromotedAsPrimary() throws Exception { for (IndexShard shard : shards.getReplicas()) { assertDocCounts(shard, totalDocs, numDocs); } - assertEquals(additonalDocs, nextPrimary.translogStats().estimatedNumberOfOperations()); - assertEquals(additonalDocs, replica.translogStats().estimatedNumberOfOperations()); - assertEquals(additonalDocs, nextPrimary.translogStats().getUncommittedOperations()); - assertEquals(additonalDocs, replica.translogStats().getUncommittedOperations()); + assertEquals(totalDocs, oldPrimary.translogStats().estimatedNumberOfOperations()); + assertEquals(totalDocs, oldPrimary.translogStats().estimatedNumberOfOperations()); + assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations()); + assertEquals(totalDocs, replica.translogStats().estimatedNumberOfOperations()); + assertEquals(totalDocs, nextPrimary.translogStats().getUncommittedOperations()); + assertEquals(totalDocs, replica.translogStats().getUncommittedOperations()); // promote the replica shards.syncGlobalCheckpoint(); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 33c748f46bd86..ff251f42ab21b 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -54,16 +54,12 @@ public void testReplicaShardRecoveryUptoLastFlushedCommit() throws Exception { assertEquals(1, primary.getRetentionLeases().leases().size()); assertFalse(primary.getRetentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replica1.routingEntry()))); - // Step 6 - Start new replica, recovery happens, and check that new replica has docs upto last flush + // Step 6 - Start new replica, recovery happens, and check that new replica has all docs final IndexShard replica2 = shards.addReplica(); shards.startAll(); - assertDocCount(replica2, numDocs); - - // Step 7 - Segment replication, check all shards have same number of docs - replicateSegments(primary, shards.getReplicas()); shards.assertAllEqual(numDocs + moreDocs); - // Step 8 - Check retention lease does not exist for the replica shard + // Step 7 - Check retention lease does not exist for the replica shard assertEquals(1, primary.getRetentionLeases().leases().size()); assertFalse(primary.getRetentionLeases().contains(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replica2.routingEntry()))); } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java index cde5cd980a91d..b5d8b2baf40dc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java @@ -20,6 +20,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.CancellableThreads; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.StoreFileMetadata; @@ -51,7 +52,7 @@ public void setUp() throws Exception { super.setUp(); final Settings settings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT").put(Settings.EMPTY).build(); primary = newStartedShard(true, settings); - replica = newShard(primary.shardId(), false); + replica = newShard(false, settings, new NRTReplicationEngineFactory()); recoverReplica(replica, primary, true); replicaDiscoveryNode = replica.recoveryState().getTargetNode(); } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 25625fbf68a64..f6185cc071941 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -13,7 +13,6 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.CancellableThreads; import org.opensearch.index.engine.NRTReplicationEngineFactory; @@ -62,10 +61,9 @@ public void setUp() throws Exception { .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put("node.name", SegmentReplicationTargetServiceTests.class.getSimpleName()) .build(); - final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); primaryShard = newStartedShard(true, settings); replicaShard = newShard(false, settings, new NRTReplicationEngineFactory()); - recoverReplica(replicaShard, primaryShard, true); + recoverReplica(replicaShard, primaryShard, true, getReplicationFunc(replicaShard)); checkpoint = new ReplicationCheckpoint(replicaShard.shardId(), 0L, 0L, 0L, 0L); SegmentReplicationSourceFactory replicationSourceFactory = mock(SegmentReplicationSourceFactory.class); replicationSource = mock(SegmentReplicationSource.class); diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index 6b0f50d80fba2..8df57ccad85cc 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -545,7 +545,7 @@ public void recoverReplica( markAsRecovering, inSyncIds, routingTable, - (a) -> null + getReplicationFunc(replica) ); OpenSearchIndexLevelReplicationTestCase.this.startReplicaAfterRecovery(replica, primary, inSyncIds, routingTable); computeReplicationTargets(); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 954e36bb38116..1fa619012ff16 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -847,7 +847,7 @@ protected DiscoveryNode getFakeDiscoNode(String id) { } protected void recoverReplica(IndexShard replica, IndexShard primary, boolean startReplica) throws IOException { - recoverReplica(replica, primary, startReplica, (a) -> null); + recoverReplica(replica, primary, startReplica, getReplicationFunc(replica)); } /** recovers a replica from the given primary **/ @@ -877,6 +877,18 @@ protected void recoverReplica( recoverReplica(replica, primary, targetSupplier, markAsRecovering, markAsStarted, (a) -> null); } + public Function, List> getReplicationFunc(final IndexShard target) { + return target.indexSettings().isSegRepEnabled() ? (shardList) -> { + try { + assert shardList.size() >= 2; + final IndexShard primary = shardList.get(0); + return replicateSegments(primary, shardList.subList(1, shardList.size())); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } : (a) -> null; + } + /** recovers a replica from the given primary **/ protected void recoverReplica( final IndexShard replica,