From 4dd0c5f00f6d21d3dbc29afcb2454d927ce7817f Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Tue, 21 Feb 2023 17:48:46 -0800 Subject: [PATCH 1/3] Adding 2.7.0 to BWC versions (#6430) Signed-off-by: Kartik Ganesh --- .ci/bwcVersions | 1 + server/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index aa12cf1020510..ff5ab09f1ff12 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -15,3 +15,4 @@ BWC_VERSION: - "2.5.0" - "2.5.1" - "2.6.0" + - "2.7.0" diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 5e589ccf7ee40..a8f3846a013c2 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -90,6 +90,7 @@ public class Version implements Comparable, ToXContentFragment { // UNRELEASED public static final Version V_2_4_2 = new Version(2040299, org.apache.lucene.util.Version.LUCENE_9_4_2); public static final Version V_2_6_0 = new Version(2060099, org.apache.lucene.util.Version.LUCENE_9_5_0); + public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version CURRENT = V_3_0_0; From 8865d58317219eca4f8ac3bd58ec37cc9bab4f9c Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 22 Feb 2023 10:59:55 -0800 Subject: [PATCH 2/3] [Segment Replication] Revert primary allocation (#6439) * Revert "[Segment Replication] Add changelog entry and fix allocation test (#6250)" This reverts commit f8102d12764ad9fa71d8e9deeeaa08fff0b1e60a. * Revert "[Segment Replication] Introduce primary weight factor for primary shards distribution (#6017)" This reverts commit aa21b5f35c1befa34e8ddfceff9c14b078d2e415. --- CHANGELOG.md | 2 +- .../opensearch/common/collect/Triplet.java | 63 ---- .../SegmentReplicationAllocationIT.java | 307 ------------------ .../cluster/routing/RoutingNode.java | 17 - .../allocator/BalancedShardsAllocator.java | 67 +--- .../allocator/LocalShardsBalancer.java | 8 - .../allocation/allocator/ShardsBalancer.java | 11 +- .../org/opensearch/common/TriConsumer.java | 13 +- .../settings/AbstractScopedSettings.java | 23 -- .../common/settings/ClusterSettings.java | 1 - .../opensearch/common/settings/Setting.java | 56 ---- .../support/MultiTermsValuesSourceConfig.java | 2 +- .../support/MultiValuesSourceFieldConfig.java | 2 +- .../allocation/BalanceConfigurationTests.java | 177 +--------- .../common/settings/SettingTests.java | 127 -------- .../metrics/SumAggregatorTests.java | 2 +- 16 files changed, 25 insertions(+), 853 deletions(-) delete mode 100644 libs/common/src/main/java/org/opensearch/common/collect/Triplet.java delete mode 100644 server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c60bdc99db9..fd861806faf04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,12 +96,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add a guardrail to limit maximum number of shard on the cluster ([#6143](https://github.com/opensearch-project/OpenSearch/pull/6143)) - Add cancellation of in-flight SearchTasks based on resource consumption ([#5606](https://github.com/opensearch-project/OpenSearch/pull/5605)) - Add support for ppc64le architecture ([#5459](https://github.com/opensearch-project/OpenSearch/pull/5459)) -- [Segment Replication] Add primary weight factor for balanced primary distribution ([#6017](https://github.com/opensearch-project/OpenSearch/pull/6017)) - Add a setting to control auto release of OpenSearch managed index creation block ([#6277](https://github.com/opensearch-project/OpenSearch/pull/6277)) - Fix timeout error when adding a document to an index with extension running ([#6275](https://github.com/opensearch-project/OpenSearch/pull/6275)) - Handle translog upload during primary relocation for remote-backed indexes ([#5804](https://github.com/opensearch-project/OpenSearch/pull/5804)) - Batch translog sync/upload per x ms for remote-backed indexes ([#5854](https://github.com/opensearch-project/OpenSearch/pull/5854)) + ### Dependencies - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704)) - Bumps `reactor-netty` from 1.1.1 to 1.1.2 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/5878)) diff --git a/libs/common/src/main/java/org/opensearch/common/collect/Triplet.java b/libs/common/src/main/java/org/opensearch/common/collect/Triplet.java deleted file mode 100644 index 9b7e8f6a419a6..0000000000000 --- a/libs/common/src/main/java/org/opensearch/common/collect/Triplet.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.common.collect; - -import java.util.Objects; - -/** - * A container for 3 elements, similar to {@link org.opensearch.common.collect.Tuple} - * - * @opensearch.internal - */ -public class Triplet { - - public static Triplet tuple(V1 v1, V2 v2, V3 v3) { - return new Triplet<>(v1, v2, v3); - } - - private final V1 v1; - private final V2 v2; - - private final V3 v3; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Triplet triplet = (Triplet) o; - return Objects.equals(v1, triplet.v1) && Objects.equals(v2, triplet.v2) && Objects.equals(v3, triplet.v3); - } - - @Override - public int hashCode() { - return Objects.hash(v1, v2, v3); - } - - public Triplet(V1 v1, V2 v2, V3 v3) { - this.v1 = v1; - this.v2 = v2; - this.v3 = v3; - } - - public V1 v1() { - return v1; - } - - public V2 v2() { - return v2; - } - - public V3 v3() { - return v3; - } - - @Override - public String toString() { - return "Tuple [v1=" + v1 + ", v2=" + v2 + ", v3=" + v3 + "]"; - } -} diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java deleted file mode 100644 index fb72c4d578510..0000000000000 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.indices.replication; - -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.routing.RoutingNode; -import org.opensearch.cluster.routing.RoutingNodes; -import org.opensearch.cluster.routing.ShardRouting; -import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.IndexModule; -import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.InternalTestCluster; -import org.opensearch.test.OpenSearchIntegTestCase; - -import java.util.ArrayList; -import java.util.Formatter; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; - -import static org.opensearch.cluster.routing.ShardRoutingState.STARTED; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; - -@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) -public class SegmentReplicationAllocationIT extends SegmentReplicationBaseIT { - - private void createIndex(String idxName, int shardCount, int replicaCount, boolean isSegRep) { - Settings.Builder builder = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount) - .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount); - if (isSegRep) { - builder = builder.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); - } else { - builder = builder.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); - } - prepareCreate(idxName, builder).get(); - } - - /** - * This test verifies primary shard allocation is balanced. - */ - public void testShardAllocation() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - final int maxReplicaCount = 2; - final int maxShardCount = 5; - final int nodeCount = randomIntBetween(maxReplicaCount + 1, 10); - final int numberOfIndices = randomIntBetween(5, 10); - - final List nodeNames = new ArrayList<>(); - logger.info("--> Creating {} nodes", nodeCount); - for (int i = 0; i < nodeCount; i++) { - nodeNames.add(internalCluster().startNode()); - } - assertAcked( - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings( - Settings.builder().put(BalancedShardsAllocator.PRIMARY_SHARD_BALANCE_FACTOR_SETTING.getKey(), "1.0f") - ) - ); - - int shardCount, replicaCount, totalShardCount = 0, totalReplicaCount = 0; - ShardAllocations shardAllocations = new ShardAllocations(); - ClusterState state; - for (int i = 0; i < numberOfIndices; i++) { - shardCount = randomIntBetween(1, maxShardCount); - totalShardCount += shardCount; - replicaCount = randomIntBetween(0, maxReplicaCount); - totalReplicaCount += replicaCount; - createIndex("test" + i, shardCount, replicaCount, i % 2 == 0); - logger.info("--> Creating index {} with shard count {} and replica count {}", "test" + i, shardCount, replicaCount); - assertBusy(() -> ensureGreen(), 60, TimeUnit.SECONDS); - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - shardAllocations.printShardDistribution(state); - } - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - RoutingNodes nodes = state.getRoutingNodes(); - final float avgNumShards = (float) (totalShardCount) / (float) (nodes.size()); - final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - 1.0f))); - final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + 1.0f))); - - for (RoutingNode node : nodes) { - assertTrue(node.primaryShardsWithState(STARTED).size() >= minAvgNumberOfShards); - assertTrue(node.primaryShardsWithState(STARTED).size() <= maxAvgNumberOfShards); - } - } - - /** - * This test verifies shard allocation with changes to cluster config i.e. node add, removal keeps the primary shard - * allocation balanced. - */ - public void testAllocationWithDisruption() throws Exception { - internalCluster().startClusterManagerOnlyNode(); - final int maxReplicaCount = 2; - final int maxShardCount = 5; - final int nodeCount = randomIntBetween(maxReplicaCount + 1, 10); - final int numberOfIndices = randomIntBetween(1, 10); - - logger.info("--> Creating {} nodes", nodeCount); - final List nodeNames = new ArrayList<>(); - for (int i = 0; i < nodeCount; i++) { - nodeNames.add(internalCluster().startNode()); - } - assertAcked( - client().admin() - .cluster() - .prepareUpdateSettings() - .setPersistentSettings( - Settings.builder() - .put(BalancedShardsAllocator.PRIMARY_SHARD_BALANCE_FACTOR_SETTING.getKey(), "1.0f") - .put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), "0.0f") - .put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), "0.0f") - .build() - ) - ); - - int shardCount, replicaCount, totalShardCount = 0, totalReplicaCount = 0; - ShardAllocations shardAllocations = new ShardAllocations(); - ClusterState state; - for (int i = 0; i < numberOfIndices; i++) { - shardCount = randomIntBetween(1, maxShardCount); - totalShardCount += shardCount; - replicaCount = randomIntBetween(1, maxReplicaCount); - totalReplicaCount += replicaCount; - logger.info("--> Creating index test{} with primary {} and replica {}", i, shardCount, replicaCount); - createIndex("test" + i, shardCount, replicaCount, i % 2 == 0); - assertBusy(() -> ensureGreen(), 60, TimeUnit.SECONDS); - if (logger.isTraceEnabled()) { - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - shardAllocations.printShardDistribution(state); - } - } - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - float avgNumShards = (float) (totalShardCount) / (float) (state.getRoutingNodes().size()); - int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - 1.0f))); - int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + 1.0f))); - - for (RoutingNode node : state.getRoutingNodes()) { - assertTrue(node.primaryShardsWithState(STARTED).size() >= minAvgNumberOfShards); - assertTrue(node.primaryShardsWithState(STARTED).size() <= maxAvgNumberOfShards); - } - - final int additionalNodeCount = randomIntBetween(1, 5); - logger.info("--> Adding {} nodes", additionalNodeCount); - - internalCluster().startNodes(additionalNodeCount); - assertBusy(() -> ensureGreen(), 60, TimeUnit.SECONDS); - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - avgNumShards = (float) (totalShardCount) / (float) (state.getRoutingNodes().size()); - minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - 1.0f))); - maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + 1.0f))); - shardAllocations.printShardDistribution(state); - for (RoutingNode node : state.getRoutingNodes()) { - assertTrue(node.primaryShardsWithState(STARTED).size() >= minAvgNumberOfShards); - assertTrue(node.primaryShardsWithState(STARTED).size() <= maxAvgNumberOfShards); - } - - logger.info("--> Stop one third nodes"); - for (int i = 1; i < nodeCount; i += 3) { - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeNames.get(i))); - // give replica a chance to promote as primary before terminating node containing the replica - assertBusy(() -> ensureGreen(), 60, TimeUnit.SECONDS); - } - state = client().admin().cluster().prepareState().execute().actionGet().getState(); - avgNumShards = (float) (totalShardCount) / (float) (state.getRoutingNodes().size()); - minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - 1.0f))); - maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + 1.0f))); - shardAllocations.printShardDistribution(state); - - for (RoutingNode node : state.getRoutingNodes()) { - assertTrue(node.primaryShardsWithState(STARTED).size() >= minAvgNumberOfShards); - assertTrue(node.primaryShardsWithState(STARTED).size() <= maxAvgNumberOfShards); - } - } - - /** - * This class is created for debugging purpose to show shard allocation across nodes. It keeps cluster state which - * is used to build the node's shard allocation - */ - private class ShardAllocations { - ClusterState state; - - public static final String separator = "==================================================="; - public static final String ONE_LINE_RETURN = "\n"; - public static final String TWO_LINE_RETURN = "\n\n"; - - /** - Store shard primary/replica shard count against a node for segrep indices. - String: NodeId - int[]: tuple storing primary shard count in 0th index and replica's in 1 - */ - TreeMap nodeToSegRepCountMap = new TreeMap<>(); - /** - Store shard primary/replica shard count against a node for docrep indices. - String: NodeId - int[]: tuple storing primary shard count in 0th index and replica's in 1 - */ - TreeMap nodeToDocRepCountMap = new TreeMap<>(); - - /** - * Helper map containing NodeName to NodeId - */ - TreeMap nameToNodeId = new TreeMap<>(); - - /* - Unassigned array containing primary at 0, replica at 1 - */ - int[] unassigned = new int[2]; - - int[] totalShards = new int[2]; - - public final String printShardAllocationWithHeader(int[] docrep, int[] segrep) { - StringBuffer sb = new StringBuffer(); - Formatter formatter = new Formatter(sb, Locale.getDefault()); - formatter.format("%-20s %-20s %-20s %-20s\n", "P", docrep[0] + segrep[0], docrep[0], segrep[0]); - formatter.format("%-20s %-20s %-20s %-20s\n", "R", docrep[1] + segrep[1], docrep[1], segrep[1]); - return sb.toString(); - } - - public void reset() { - nodeToSegRepCountMap.clear(); - nodeToDocRepCountMap.clear(); - nameToNodeId.clear(); - totalShards[0] = totalShards[1] = 0; - unassigned[0] = unassigned[1] = 0; - } - - public void setState(ClusterState state) { - this.reset(); - this.state = state; - buildMap(); - } - - private void buildMap() { - for (RoutingNode node : state.getRoutingNodes()) { - nameToNodeId.putIfAbsent(node.node().getName(), node.nodeId()); - nodeToSegRepCountMap.putIfAbsent(node.nodeId(), new int[] { 0, 0 }); - nodeToDocRepCountMap.putIfAbsent(node.nodeId(), new int[] { 0, 0 }); - } - for (ShardRouting shardRouting : state.routingTable().allShards()) { - // Fetch shard to update. Initialize local array - if (isIndexSegRep(shardRouting.getIndexName())) { - updateMap(nodeToSegRepCountMap, shardRouting); - } else { - updateMap(nodeToDocRepCountMap, shardRouting); - } - } - } - - void updateMap(TreeMap mapToUpdate, ShardRouting shardRouting) { - int[] shard; - shard = shardRouting.assignedToNode() ? mapToUpdate.get(shardRouting.currentNodeId()) : unassigned; - // Update shard type count - if (shardRouting.primary()) { - shard[0]++; - totalShards[0]++; - } else { - shard[1]++; - totalShards[1]++; - } - // For assigned shards, put back counter - if (shardRouting.assignedToNode()) mapToUpdate.put(shardRouting.currentNodeId(), shard); - } - - boolean isIndexSegRep(String indexName) { - return state.metadata() - .index(indexName) - .getSettings() - .get(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey()) - .equals(ReplicationType.SEGMENT.toString()); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append(TWO_LINE_RETURN + separator + ONE_LINE_RETURN); - Formatter formatter = new Formatter(sb, Locale.getDefault()); - for (Map.Entry entry : nameToNodeId.entrySet()) { - String nodeId = nameToNodeId.get(entry.getKey()); - formatter.format("%-20s %-20s %-20s %-20s\n", entry.getKey().toUpperCase(Locale.getDefault()), "TOTAL", "DOCREP", "SEGREP"); - sb.append(printShardAllocationWithHeader(nodeToDocRepCountMap.get(nodeId), nodeToSegRepCountMap.get(nodeId))); - } - sb.append(ONE_LINE_RETURN); - formatter.format("%-20s %-20s %-20s\n\n", "Unassigned ", unassigned[0], unassigned[1]); - formatter.format("%-20s %-20s %-20s\n\n", "Total Shards", totalShards[0], totalShards[1]); - return sb.toString(); - } - - public void printShardDistribution(ClusterState state) { - this.setState(state); - logger.info("--> Shard distribution {}", this); - } - } - -} diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java index d23b4856eced0..413ddff72f7a5 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java @@ -329,23 +329,6 @@ public List shardsWithState(ShardRoutingState... states) { return shards; } - /** - * Determine the primary shards of an index with a specific state - * @param states set of states which should be listed - * @return a list of shards - */ - public List primaryShardsWithState(ShardRoutingState... states) { - List shards = new ArrayList<>(); - for (ShardRouting shardEntry : this) { - for (ShardRoutingState state : states) { - if (shardEntry.state() == state && shardEntry.primary() == true) { - shards.add(shardEntry); - } - } - } - return shards; - } - /** * Determine the shards of an index with a specific state * @param index id of the index diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 8893aaa54799a..d8761e9b1a78e 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -107,14 +107,6 @@ public class BalancedShardsAllocator implements ShardsAllocator { Property.NodeScope ); - public static final Setting PRIMARY_SHARD_BALANCE_FACTOR_SETTING = Setting.floatSetting( - "cluster.routing.allocation.balance.primary", - 0.0f, - 0.0f, - Property.Dynamic, - Property.NodeScope - ); - private volatile boolean movePrimaryFirst; private volatile WeightFunction weightFunction; private volatile float threshold; @@ -125,19 +117,10 @@ public BalancedShardsAllocator(Settings settings) { @Inject public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings) { - setWeightFunction( - INDEX_BALANCE_FACTOR_SETTING.get(settings), - SHARD_BALANCE_FACTOR_SETTING.get(settings), - PRIMARY_SHARD_BALANCE_FACTOR_SETTING.get(settings) - ); + setWeightFunction(INDEX_BALANCE_FACTOR_SETTING.get(settings), SHARD_BALANCE_FACTOR_SETTING.get(settings)); setThreshold(THRESHOLD_SETTING.get(settings)); clusterSettings.addSettingsUpdateConsumer(SHARD_MOVE_PRIMARY_FIRST_SETTING, this::setMovePrimaryFirst); - clusterSettings.addSettingsUpdateConsumer( - INDEX_BALANCE_FACTOR_SETTING, - SHARD_BALANCE_FACTOR_SETTING, - PRIMARY_SHARD_BALANCE_FACTOR_SETTING, - this::setWeightFunction - ); + clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, SHARD_BALANCE_FACTOR_SETTING, this::setWeightFunction); clusterSettings.addSettingsUpdateConsumer(THRESHOLD_SETTING, this::setThreshold); } @@ -145,8 +128,8 @@ private void setMovePrimaryFirst(boolean movePrimaryFirst) { this.movePrimaryFirst = movePrimaryFirst; } - private void setWeightFunction(float indexBalance, float shardBalanceFactor, float primaryShardBalance) { - weightFunction = new WeightFunction(indexBalance, shardBalanceFactor, primaryShardBalance); + private void setWeightFunction(float indexBalance, float shardBalanceFactor) { + weightFunction = new WeightFunction(indexBalance, shardBalanceFactor); } private void setThreshold(float threshold) { @@ -269,22 +252,17 @@ static class WeightFunction { private final float shardBalance; private final float theta0; private final float theta1; - private final float theta2; - private final float primaryShardBalance; private AllocationConstraints constraints; - WeightFunction(float indexBalance, float shardBalance, float primaryShardBalance) { - float sum = indexBalance + shardBalance + primaryShardBalance; + WeightFunction(float indexBalance, float shardBalance) { + float sum = indexBalance + shardBalance; if (sum <= 0.0f) { throw new IllegalArgumentException("Balance factors must sum to a value > 0 but was: " + sum); } theta0 = shardBalance / sum; theta1 = indexBalance / sum; - theta2 = primaryShardBalance / sum; - this.indexBalance = indexBalance; this.shardBalance = shardBalance; - this.primaryShardBalance = primaryShardBalance; this.constraints = new AllocationConstraints(); } @@ -296,9 +274,7 @@ public float weightWithAllocationConstraints(ShardsBalancer balancer, ModelNode float weight(ShardsBalancer balancer, ModelNode node, String index) { final float weightShard = node.numShards() - balancer.avgShardsPerNode(); final float weightIndex = node.numShards(index) - balancer.avgShardsPerNode(index); - final float primaryWeightShard = node.numPrimaryShards() - balancer.avgPrimaryShardsPerNode(); - - return theta0 * weightShard + theta1 * weightIndex + theta2 * primaryWeightShard; + return theta0 * weightShard + theta1 * weightIndex; } } @@ -312,8 +288,6 @@ public static class ModelNode implements Iterable { private int numShards = 0; private final RoutingNode routingNode; - private int primaryNumShards = 0; - ModelNode(RoutingNode routingNode) { this.routingNode = routingNode; } @@ -339,10 +313,6 @@ public int numShards(String idx) { return index == null ? 0 : index.numShards(); } - public int numPrimaryShards() { - return primaryNumShards; - } - public int highestPrimary(String index) { ModelIndex idx = indices.get(index); if (idx != null) { @@ -359,9 +329,6 @@ public void addShard(ShardRouting shard) { } index.addShard(shard); numShards++; - if (shard.primary()) { - primaryNumShards++; - } } public void removeShard(ShardRouting shard) { @@ -372,9 +339,6 @@ public void removeShard(ShardRouting shard) { indices.remove(shard.getIndexName()); } } - if (shard.primary()) { - primaryNumShards--; - } numShards--; } @@ -417,14 +381,13 @@ public Balancer( } /** - * A model index that stores info about specific index + * A model index. * * @opensearch.internal */ static final class ModelIndex implements Iterable { private final String id; private final Set shards = new HashSet<>(4); // expect few shards of same index to be allocated on same node - private final Set primaryShards = new HashSet<>(); private int highestPrimary = -1; ModelIndex(String id) { @@ -452,10 +415,6 @@ public int numShards() { return shards.size(); } - public int numPrimaryShards() { - return primaryShards.size(); - } - @Override public Iterator iterator() { return shards.iterator(); @@ -464,20 +423,12 @@ public Iterator iterator() { public void removeShard(ShardRouting shard) { highestPrimary = -1; assert shards.contains(shard) : "Shard not allocated on current node: " + shard; - if (shard.primary()) { - assert primaryShards.contains(shard) : "Primary shard not allocated on current node: " + shard; - primaryShards.remove(shard); - } shards.remove(shard); } public void addShard(ShardRouting shard) { highestPrimary = -1; - assert shards.contains(shard) == false : "Shard already allocated on current node: " + shard; - if (shard.primary()) { - assert primaryShards.contains(shard) == false : "Primary shard already allocated on current node: " + shard; - primaryShards.add(shard); - } + assert !shards.contains(shard) : "Shard already allocated on current node: " + shard; shards.add(shard); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java index 34ad9761b3f1a..8570a16fd690c 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java @@ -63,7 +63,6 @@ public class LocalShardsBalancer extends ShardsBalancer { private final float threshold; private final Metadata metadata; private final float avgShardsPerNode; - private final float avgPrimaryShardsPerNode; private final BalancedShardsAllocator.NodeSorter sorter; private final Set inEligibleTargetNode; @@ -82,8 +81,6 @@ public LocalShardsBalancer( this.routingNodes = allocation.routingNodes(); this.metadata = allocation.metadata(); avgShardsPerNode = ((float) metadata.getTotalNumberOfShards()) / routingNodes.size(); - final int shardCount = StreamSupport.stream(metadata.spliterator(), false).mapToInt(IndexMetadata::getNumberOfShards).sum(); - avgPrimaryShardsPerNode = (float) shardCount / routingNodes.size(); nodes = Collections.unmodifiableMap(buildModelFromAssigned()); sorter = newNodeSorter(); inEligibleTargetNode = new HashSet<>(); @@ -104,11 +101,6 @@ public float avgShardsPerNode(String index) { return ((float) metadata.index(index).getTotalNumberOfShards()) / nodes.size(); } - @Override - public float avgPrimaryShardsPerNode() { - return avgPrimaryShardsPerNode; - } - /** * Returns the global average of shards per node */ diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsBalancer.java index 9774f84a4cd91..593e6998141fb 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsBalancer.java +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/ShardsBalancer.java @@ -60,23 +60,16 @@ public abstract class ShardsBalancer { abstract MoveDecision decideRebalance(ShardRouting shardRouting); /** - * Returns the average of shards per node + * Returns the average of shards per node for the given index */ public float avgShardsPerNode() { return Float.MAX_VALUE; } /** - * Returns the global average of shards per node for the given index + * Returns the global average of shards per node */ public float avgShardsPerNode(String index) { return Float.MAX_VALUE; } - - /** - * Returns the average of primary shards per node - */ - public float avgPrimaryShardsPerNode() { - return Float.MAX_VALUE; - } } diff --git a/server/src/main/java/org/opensearch/common/TriConsumer.java b/server/src/main/java/org/opensearch/common/TriConsumer.java index a174499d58628..f98276b6d007d 100644 --- a/server/src/main/java/org/opensearch/common/TriConsumer.java +++ b/server/src/main/java/org/opensearch/common/TriConsumer.java @@ -32,8 +32,6 @@ package org.opensearch.common; -import java.util.Objects; - /** * Represents an operation that accepts three arguments and returns no result. * @@ -52,14 +50,5 @@ public interface TriConsumer { * @param t the second function argument * @param u the third function argument */ - void accept(S s, T t, U u); - - default TriConsumer andThen(TriConsumer after) { - Objects.requireNonNull(after); - - return (l, r, s) -> { - accept(l, r, s); - after.accept(l, r, s); - }; - } + void apply(S s, T t, U u); } diff --git a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java index abc4ee70cb3e5..8b7a2a82e5cb1 100644 --- a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java @@ -38,7 +38,6 @@ import org.apache.lucene.search.spell.LevenshteinDistance; import org.apache.lucene.util.CollectionUtil; import org.opensearch.ExceptionsHelper; -import org.opensearch.common.TriConsumer; import org.opensearch.common.collect.Tuple; import org.opensearch.common.regex.Regex; @@ -443,28 +442,6 @@ public synchronized void addSettingsUpdateConsumer(Setting a, Setting< addSettingsUpdateConsumer(a, b, consumer, (i, j) -> {}); } - /** - * Adds a settings consumer that accepts the values for three settings. The consumer is only notified if any one of - * the settings changed and if the provided validator succeeded. - *

- * Note: Only settings registered in {@link SettingsModule} can be changed dynamically. - *

- * This method registers a compound updater that is useful if three settings depends on each other. - * The consumer is always provided with both values even if only one of the two changes. - */ - public synchronized void addSettingsUpdateConsumer(Setting a, Setting b, Setting c, TriConsumer consumer) { - if (a != get(a.getKey())) { - throw new IllegalArgumentException("Setting is not registered for key [" + a.getKey() + "]"); - } - if (b != get(b.getKey())) { - throw new IllegalArgumentException("Setting is not registered for key [" + b.getKey() + "]"); - } - if (c != get(c.getKey())) { - throw new IllegalArgumentException("Setting is not registered for key [" + c.getKey() + "]"); - } - addSettingsUpdater(Setting.compoundUpdater(consumer, (i, j, k) -> {}, a, b, c, logger)); - } - /** * Adds a settings consumer that accepts the values for two settings. The consumer is only notified if one or both settings change * and if the provided validator succeeded. diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 37fd8d1bb594a..475b73a55cc51 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -230,7 +230,6 @@ public void apply(Settings value, Settings current, Settings previous) { AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING, BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING, BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING, - BalancedShardsAllocator.PRIMARY_SHARD_BALANCE_FACTOR_SETTING, BalancedShardsAllocator.SHARD_MOVE_PRIMARY_FIRST_SETTING, BalancedShardsAllocator.THRESHOLD_SETTING, BreakerSettings.CIRCUIT_BREAKER_LIMIT_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index ceec489b1f57f..556b60b222716 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -39,8 +39,6 @@ import org.opensearch.common.Booleans; import org.opensearch.common.Nullable; import org.opensearch.common.Strings; -import org.opensearch.common.TriConsumer; -import org.opensearch.common.collect.Triplet; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -727,60 +725,6 @@ public String toString() { }; } - /** - * Updates settings that depend on each other. - * - * See {@link AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, Setting, TriConsumer)} and its usage for details. - */ - static AbstractScopedSettings.SettingUpdater> compoundUpdater( - final TriConsumer consumer, - final TriConsumer validator, - final Setting aSetting, - final Setting bSetting, - final Setting cSetting, - Logger logger - ) { - final AbstractScopedSettings.SettingUpdater aSettingUpdater = aSetting.newUpdater(null, logger); - final AbstractScopedSettings.SettingUpdater bSettingUpdater = bSetting.newUpdater(null, logger); - final AbstractScopedSettings.SettingUpdater cSettingUpdater = cSetting.newUpdater(null, logger); - return new AbstractScopedSettings.SettingUpdater>() { - @Override - public boolean hasChanged(Settings current, Settings previous) { - return aSettingUpdater.hasChanged(current, previous) - || bSettingUpdater.hasChanged(current, previous) - || cSettingUpdater.hasChanged(current, previous); - } - - @Override - public Triplet getValue(Settings current, Settings previous) { - A valueA = aSettingUpdater.getValue(current, previous); - B valueB = bSettingUpdater.getValue(current, previous); - C valueC = cSettingUpdater.getValue(current, previous); - validator.accept(valueA, valueB, valueC); - return new Triplet<>(valueA, valueB, valueC); - } - - @Override - public void apply(Triplet value, Settings current, Settings previous) { - if (aSettingUpdater.hasChanged(current, previous)) { - logSettingUpdate(aSetting, current, previous, logger); - } - if (bSettingUpdater.hasChanged(current, previous)) { - logSettingUpdate(bSetting, current, previous, logger); - } - if (cSettingUpdater.hasChanged(current, previous)) { - logSettingUpdate(cSetting, current, previous, logger); - } - consumer.accept(value.v1(), value.v2(), value.v3()); - } - - @Override - public String toString() { - return "CompoundUpdater for: " + aSettingUpdater + " and " + bSettingUpdater + " and " + cSettingUpdater; - } - }; - } - static AbstractScopedSettings.SettingUpdater groupedSettingsUpdater( Consumer consumer, final List> configuredSettings diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java index 5905c7a0b21d8..4e6beefd1ee88 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiTermsValuesSourceConfig.java @@ -57,7 +57,7 @@ ObjectParser apply( MultiTermsValuesSourceConfig.Builder::new ); - BaseMultiValuesSourceFieldConfig.PARSER.accept(parser, scriptable, timezoneAware); + BaseMultiValuesSourceFieldConfig.PARSER.apply(parser, scriptable, timezoneAware); if (valueTypeHinted) { parser.declareField( diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java index 6ba4d4b5b21f1..3862e5ebe9d1a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/MultiValuesSourceFieldConfig.java @@ -68,7 +68,7 @@ public class MultiValuesSourceFieldConfig extends BaseMultiValuesSourceFieldConf MultiValuesSourceFieldConfig.Builder::new ); - BaseMultiValuesSourceFieldConfig.PARSER.accept(parser, scriptable, timezoneAware); + BaseMultiValuesSourceFieldConfig.PARSER.apply(parser, scriptable, timezoneAware); if (filtered) { parser.declareField( diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java index 3762e137ac8da..1ba69694eaec1 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java @@ -75,7 +75,7 @@ public class BalanceConfigurationTests extends OpenSearchAllocationTestCase { public void testIndexBalance() { /* Tests balance over indices only */ final float indexBalance = 1.0f; - final float shardBalance = 0.0f; + final float replicaBalance = 0.0f; final float balanceThreshold = 1.0f; Settings.Builder settings = Settings.builder(); @@ -84,7 +84,7 @@ public void testIndexBalance() { ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString() ); settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance); - settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance); + settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance); settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold); AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator()); @@ -123,148 +123,10 @@ public void testIndexBalance() { ); } - /** - * This test verifies that with only primary shard balance, the primary shard distribution is balanced within thresholds. - */ - public void testPrimaryBalance() { - /* Tests balance over primary shards only */ - final float indexBalance = 0.0f; - final float shardBalance = 0.0f; - final float primaryBalance = 1.0f; - final float balanceThreshold = 1.0f; - - Settings.Builder settings = Settings.builder(); - settings.put( - ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), - ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString() - ); - settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance); - settings.put(BalancedShardsAllocator.PRIMARY_SHARD_BALANCE_FACTOR_SETTING.getKey(), primaryBalance); - settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance); - settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold); - - AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator()); - - ClusterState clusterState = initCluster(strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - - clusterState = addNode(clusterState, strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes + 1, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - - clusterState = removeNodes(clusterState, strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - (numberOfNodes + 1) - (numberOfNodes + 1) / 2, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - } - - /** - * This test verifies - */ - public void testBalanceDefaults() { - final float indexBalance = 0.55f; - final float shardBalance = 0.45f; - final float primaryBalance = 0.40f; - final float balanceThreshold = 1.0f; - - Settings.Builder settings = Settings.builder(); - settings.put( - ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), - ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString() - ); - settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance); - settings.put(BalancedShardsAllocator.PRIMARY_SHARD_BALANCE_FACTOR_SETTING.getKey(), primaryBalance); - settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance); - settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold); - - AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator()); - - ClusterState clusterState = initCluster(strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - assertIndexBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - - clusterState = addNode(clusterState, strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes + 1, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - assertIndexBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - numberOfNodes + 1, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - - clusterState = removeNodes(clusterState, strategy); - assertPrimaryBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - (numberOfNodes + 1) - (numberOfNodes + 1) / 2, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - assertIndexBalance( - clusterState.getRoutingTable(), - clusterState.getRoutingNodes(), - (numberOfNodes + 1) - (numberOfNodes + 1) / 2, - numberOfIndices, - numberOfReplicas, - numberOfShards, - balanceThreshold - ); - } - - public void testShardBalance() { + public void testReplicaBalance() { /* Tests balance over replicas only */ final float indexBalance = 0.0f; - final float shardBalance = 1.0f; + final float replicaBalance = 1.0f; final float balanceThreshold = 1.0f; Settings.Builder settings = Settings.builder(); @@ -273,13 +135,13 @@ public void testShardBalance() { ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString() ); settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance); - settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance); + settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), replicaBalance); settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold); AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator()); ClusterState clusterState = initCluster(strategy); - assertShardBalance( + assertReplicaBalance( clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices, @@ -289,7 +151,7 @@ public void testShardBalance() { ); clusterState = addNode(clusterState, strategy); - assertShardBalance( + assertReplicaBalance( clusterState.getRoutingNodes(), numberOfNodes + 1, numberOfIndices, @@ -299,7 +161,7 @@ public void testShardBalance() { ); clusterState = removeNodes(clusterState, strategy); - assertShardBalance( + assertReplicaBalance( clusterState.getRoutingNodes(), numberOfNodes + 1 - (numberOfNodes + 1) / 2, numberOfIndices, @@ -392,7 +254,7 @@ private ClusterState removeNodes(ClusterState clusterState, AllocationService st return applyStartedShardsUntilNoChange(clusterState, strategy); } - private void assertShardBalance( + private void assertReplicaBalance( RoutingNodes nodes, int numberOfNodes, int numberOfIndices, @@ -447,27 +309,6 @@ private void assertIndexBalance( } } - private void assertPrimaryBalance( - RoutingTable routingTable, - RoutingNodes nodes, - int numberOfNodes, - int numberOfIndices, - int numberOfReplicas, - int numberOfShards, - float threshold - ) { - - final int numShards = numberOfShards * numberOfIndices; - final float avgNumShards = (float) (numShards) / (float) (numberOfNodes); - final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - threshold))); - final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + threshold))); - - for (RoutingNode node : nodes) { - assertThat(node.primaryShardsWithState(STARTED).size(), Matchers.greaterThanOrEqualTo(minAvgNumberOfShards)); - assertThat(node.primaryShardsWithState(STARTED).size(), Matchers.lessThanOrEqualTo(maxAvgNumberOfShards)); - } - } - public void testPersistedSettings() { Settings.Builder settings = Settings.builder(); settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), 0.2); diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index e102e0aa9b6a4..d7633d4162437 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -36,7 +36,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LogEvent; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.collect.Triplet; import org.opensearch.common.collect.Tuple; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.settings.AbstractScopedSettings.SettingUpdater; @@ -637,28 +636,6 @@ public void validate(Integer a, Integer b) { } } - // This test class is used to verify behavior of BalancedShardAllocator.WeightFunction and ensure set function is called - // whenever there is a change in any of the settings. - public static class TriSettingConsumer { - - private Integer b; - private Integer a; - - private Integer c; - - public void set(Integer a, Integer b, Integer c) { - this.a = a; - this.b = b; - this.c = c; - } - - public void validate(Integer a, Integer b, Integer c) { - if (Integer.signum(a) != Integer.signum(b) || Integer.signum(a) != Integer.signum(c)) { - throw new IllegalArgumentException("boom"); - } - } - } - public void testComposite() { Composite c = new Composite(); Setting a = Setting.intSetting("foo.int.bar.a", 1, Property.Dynamic, Property.NodeScope); @@ -723,110 +700,6 @@ public void testCompositeValidator() { } - public void testTriSettingConsumer() { - TriSettingConsumer consumer = new TriSettingConsumer(); - Setting a = Setting.intSetting("foo.int.bar.a", 1, Property.Dynamic, Property.NodeScope); - Setting b = Setting.intSetting("foo.int.bar.b", 1, Property.Dynamic, Property.NodeScope); - Setting c = Setting.intSetting("foo.int.bar.c", 1, Property.Dynamic, Property.NodeScope); - ClusterSettings.SettingUpdater> settingUpdater = Setting.compoundUpdater( - consumer::set, - consumer::validate, - a, - b, - c, - logger - ); - assertFalse(settingUpdater.apply(Settings.EMPTY, Settings.EMPTY)); - assertNull(consumer.a); - assertNull(consumer.b); - assertNull(consumer.c); - - Settings build = Settings.builder().put("foo.int.bar.a", 2).build(); - assertTrue(settingUpdater.apply(build, Settings.EMPTY)); - assertEquals(2, consumer.a.intValue()); - assertEquals(1, consumer.b.intValue()); - assertEquals(1, consumer.c.intValue()); - - Integer aValue = consumer.a; - assertFalse(settingUpdater.apply(build, build)); - assertSame(aValue, consumer.a); - Settings previous = build; - build = Settings.builder().put("foo.int.bar.a", 2).put("foo.int.bar.b", 5).build(); - assertTrue(settingUpdater.apply(build, previous)); - assertEquals(2, consumer.a.intValue()); - assertEquals(5, consumer.b.intValue()); - - Integer bValue = consumer.b; - assertFalse(settingUpdater.apply(build, build)); - assertSame(bValue, consumer.b); - previous = build; - build = Settings.builder().put("foo.int.bar.a", 2).put("foo.int.bar.b", 5).put("foo.int.bar.c", 10).build(); - assertTrue(settingUpdater.apply(build, previous)); - assertEquals(2, consumer.a.intValue()); - assertEquals(5, consumer.b.intValue()); - assertEquals(10, consumer.c.intValue()); - - // reset to default - assertTrue(settingUpdater.apply(Settings.EMPTY, build)); - assertEquals(1, consumer.a.intValue()); - assertEquals(1, consumer.b.intValue()); - assertEquals(1, consumer.c.intValue()); - } - - public void testTriSettingConsumerValidator() { - TriSettingConsumer consumer = new TriSettingConsumer(); - Setting a = Setting.intSetting("foo.int.bar.a", 1, Property.Dynamic, Property.NodeScope); - Setting b = Setting.intSetting("foo.int.bar.b", 1, Property.Dynamic, Property.NodeScope); - Setting c = Setting.intSetting("foo.int.bar.c", 1, Property.Dynamic, Property.NodeScope); - ClusterSettings.SettingUpdater> settingUpdater = Setting.compoundUpdater( - consumer::set, - consumer::validate, - a, - b, - c, - logger - ); - assertFalse(settingUpdater.apply(Settings.EMPTY, Settings.EMPTY)); - assertNull(consumer.a); - assertNull(consumer.b); - assertNull(consumer.c); - - Settings build = Settings.builder().put("foo.int.bar.a", 2).build(); - assertTrue(settingUpdater.apply(build, Settings.EMPTY)); - assertEquals(2, consumer.a.intValue()); - assertEquals(1, consumer.b.intValue()); - assertEquals(1, consumer.c.intValue()); - - Integer aValue = consumer.a; - assertFalse(settingUpdater.apply(build, build)); - assertSame(aValue, consumer.a); - final Settings previous = build; - build = Settings.builder().put("foo.int.bar.a", 2).put("foo.int.bar.b", 5).build(); - assertTrue(settingUpdater.apply(build, previous)); - assertEquals(2, consumer.a.intValue()); - assertEquals(5, consumer.b.intValue()); - - Integer bValue = consumer.b; - assertFalse(settingUpdater.apply(build, build)); - assertSame(bValue, consumer.b); - final Settings previous2 = build; - build = Settings.builder().put("foo.int.bar.a", 2).put("foo.int.bar.b", 5).put("foo.int.bar.c", 10).build(); - assertTrue(settingUpdater.apply(build, previous)); - assertEquals(2, consumer.a.intValue()); - assertEquals(5, consumer.b.intValue()); - assertEquals(10, consumer.c.intValue()); - - Settings invalid = Settings.builder().put("foo.int.bar.a", -2).put("foo.int.bar.b", 5).build(); - IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> settingUpdater.apply(invalid, previous2)); - assertThat(exc.getMessage(), equalTo("boom")); - - // reset to default - assertTrue(settingUpdater.apply(Settings.EMPTY, build)); - assertEquals(1, consumer.a.intValue()); - assertEquals(1, consumer.b.intValue()); - assertEquals(1, consumer.c.intValue()); - } - public void testListSettingsDeprecated() { final Setting> deprecatedListSetting = Setting.listSetting( "foo.deprecated", diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java index b38d4eee850ef..8c0087ca0b87d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/metrics/SumAggregatorTests.java @@ -385,7 +385,7 @@ private void sumRandomDocsTestCase( builder, new MatchAllDocsQuery(), writer -> writer.addDocuments(docs), - internalSum -> verify.accept(finalSum, docs, internalSum), + internalSum -> verify.apply(finalSum, docs, internalSum), fieldType ); } From 01466a6a04e280267f013aeb14dff1f20dc7a77a Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Wed, 22 Feb 2023 12:24:45 -0800 Subject: [PATCH 3/3] Add release notes for v2.6.0 (#6443) * Add release notes for v2.6.0 This change also includes cleanup of the CHANGELOG file to only capture unreleased changes. Signed-off-by: Kartik Ganesh * Minor language updates in v2.6.0 release notes and Changelog Signed-off-by: Kartik Ganesh --------- Signed-off-by: Kartik Ganesh --- CHANGELOG.md | 98 +++++-------------- .../opensearch.release-notes-2.6.0.md | 57 +++++++++++ 2 files changed, 82 insertions(+), 73 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.6.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fd861806faf04..82c289f41fbfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,49 +10,37 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) ### Dependencies -- Bumps `log4j-core` from 2.18.0 to 2.19.0 -- Bumps `reactor-netty-http` from 1.0.24 to 1.1.2 -- Bumps `reactor-netty-http` from 1.0.18 to 1.1.2 -- Bumps `jettison` from 1.5.0 to 1.5.3 -- Bumps `forbiddenapis` from 3.3 to 3.4 -- Bumps `avro` from 1.11.0 to 1.11.1 -- Bumps `woodstox-core` from 6.3.0 to 6.3.1 -- Bumps `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354)) -- Bumps `azure-storage-common` from 12.18.0 to 12.18.1 ([#4164](https://github.com/opensearch-project/OpenSearch/pull/4664)) -- Bumps `org.gradle.test-retry` from 1.4.0 to 1.4.1 ([#4411](https://github.com/opensearch-project/OpenSearch/pull/4411)) -- Bumps `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447)) -- Bumps `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488)) -- Bumps `com.diffplug.spotless` from 6.11.0 to 6.15.0 -- Bumps `com.diffplug.spotless` from 6.10.0 to 6.15.0 -- Bumps `reactor-core` from 3.4.23 to 3.5.1 ([#5604](https://github.com/opensearch-project/OpenSearch/pull/5604)) -- Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550)) -- Bumps `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#5315](https://github.com/opensearch-project/OpenSearch/pull/5315)) +- Bump `log4j-core` from 2.18.0 to 2.19.0 +- Bump `forbiddenapis` from 3.3 to 3.4 +- Bump `avro` from 1.11.0 to 1.11.1 +- Bump `woodstox-core` from 6.3.0 to 6.3.1 +- Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354)) +- Bump `azure-storage-common` from 12.18.0 to 12.18.1 ([#4164](https://github.com/opensearch-project/OpenSearch/pull/4664)) +- Bump `org.gradle.test-retry` from 1.4.0 to 1.4.1 ([#4411](https://github.com/opensearch-project/OpenSearch/pull/4411)) +- Bump `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447)) +- Bump `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488)) +- Bump `reactor-core` from 3.4.23 to 3.5.1 ([#5604](https://github.com/opensearch-project/OpenSearch/pull/5604)) +- Bump `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550)) +- Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#5315](https://github.com/opensearch-project/OpenSearch/pull/5315)) - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) - Update Apache Lucene to 9.5.0-snapshot-d5cef1c ([#5570](https://github.com/opensearch-project/OpenSearch/pull/5570)) -- Bump antlr4 from 4.9.3 to 4.11.1 ([#4546](https://github.com/opensearch-project/OpenSearch/pull/4546)) -- Bumps `maven-model` from 3.6.2 to 3.8.6 ([#5599](https://github.com/opensearch-project/OpenSearch/pull/5599)) -- Bumps `maxmind-db` from 2.1.0 to 3.0.0 ([#5601](https://github.com/opensearch-project/OpenSearch/pull/5601)) -- Bumps `protobuf-java` from 3.21.11 to 3.21.12 ([#5603](https://github.com/opensearch-project/OpenSearch/pull/5603)) -- Bumps `azure-core-http-netty` from 1.12.7 to 1.12.8 -- Bumps `wiremock-jre8-standalone` from 2.33.2 to 2.35.0 -- Bumps `gson` from 2.10 to 2.10.1 -- Bumps `json-schema-validator` from 1.0.73 to 1.0.76 -- Bumps `jna` from 5.11.0 to 5.13.0 -- Bumps `joni` from 2.1.44 to 2.1.45 -- Bumps `commons-io:commons-io` from 2.7 to 2.11.0 -- Bumps `org.jruby.joni:joni` from 2.1.45 to 2.1.48 -- Bumps `org.apache.ant:ant` from 1.10.12 to 1.10.13 -- Bumps `com.azure:azure-storage-common` from 12.18.1 to 12.19.3 -- Bumps `io.projectreactor.netty:reactor-netty` from 1.1.2 to 1.1.3 -- Bumps `com.google.protobuf:protobuf-java` from 3.21.12 to 3.22.0 +- Bump `maven-model` from 3.6.2 to 3.8.6 ([#5599](https://github.com/opensearch-project/OpenSearch/pull/5599)) +- Bump `maxmind-db` from 2.1.0 to 3.0.0 ([#5601](https://github.com/opensearch-project/OpenSearch/pull/5601)) +- Bump `protobuf-java` from 3.21.11 to 3.21.12 ([#5603](https://github.com/opensearch-project/OpenSearch/pull/5603)) +- Bump `wiremock-jre8-standalone` from 2.33.2 to 2.35.0 +- Bump `gson` from 2.10 to 2.10.1 +- Bump `json-schema-validator` from 1.0.73 to 1.0.76 +- Bump `jna` from 5.11.0 to 5.13.0 +- Bump `joni` from 2.1.44 to 2.1.45 +- Bump `commons-io:commons-io` from 2.7 to 2.11.0 +- Bump `org.jruby.joni:joni` from 2.1.45 to 2.1.48 ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638)) - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459)) -- Changed http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) +- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773)) - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283)) -- Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009)) - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792)) ### Deprecated @@ -71,53 +59,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837)) - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018)) - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021)) -- Remove deprecated org.gradle.util.DistributionLocator usage ([#6212](https://github.com/opensearch-project/OpenSearch/pull/6212)) ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) -- Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) +- Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460)) ### Security ## [Unreleased 2.x] ### Added -- Adding index create block when all nodes have breached high disk watermark ([#5852](https://github.com/opensearch-project/OpenSearch/pull/5852)) -- Added cluster manager throttling stats in nodes/stats API ([#5790](https://github.com/opensearch-project/OpenSearch/pull/5790)) -- Added support for feature flags in opensearch.yml ([#4959](https://github.com/opensearch-project/OpenSearch/pull/4959)) -- Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658)) -- Add update-index-settings allowlist for searchable snapshot ([#5907](https://github.com/opensearch-project/OpenSearch/pull/5907)) -- Add new cat/segment_replication API to surface Segment Replication metrics ([#5718](https://github.com/opensearch-project/OpenSearch/pull/5718)). -- Replace latches with CompletableFutures for extensions ([#5646](https://github.com/opensearch-project/OpenSearch/pull/5646)) - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589)) -- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874)) -- Added support to apply index create block ([#4603](https://github.com/opensearch-project/OpenSearch/issues/4603)) -- Adds support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003)) -- Add a guardrail to limit maximum number of shard on the cluster ([#6143](https://github.com/opensearch-project/OpenSearch/pull/6143)) -- Add cancellation of in-flight SearchTasks based on resource consumption ([#5606](https://github.com/opensearch-project/OpenSearch/pull/5605)) -- Add support for ppc64le architecture ([#5459](https://github.com/opensearch-project/OpenSearch/pull/5459)) -- Add a setting to control auto release of OpenSearch managed index creation block ([#6277](https://github.com/opensearch-project/OpenSearch/pull/6277)) -- Fix timeout error when adding a document to an index with extension running ([#6275](https://github.com/opensearch-project/OpenSearch/pull/6275)) -- Handle translog upload during primary relocation for remote-backed indexes ([#5804](https://github.com/opensearch-project/OpenSearch/pull/5804)) -- Batch translog sync/upload per x ms for remote-backed indexes ([#5854](https://github.com/opensearch-project/OpenSearch/pull/5854)) - ### Dependencies -- Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704)) -- Bumps `reactor-netty` from 1.1.1 to 1.1.2 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/5878)) -- OpenJDK Update (January 2023 Patch releases) ([#6074](https://github.com/opensearch-project/OpenSearch/pull/6074)) -- Bumps `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6076](https://github.com/opensearch-project/OpenSearch/pull/6076)) -- Bumps `joda` from 2.10.13 to 2.12.2 ([#6083](https://github.com/opensearch-project/OpenSearch/pull/6083)) -- Upgrade to Lucene 9.5.0 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/6078)) -- Bumps `Jackson` from 2.14.1 to 2.14.2 ([#6129](https://github.com/opensearch-project/OpenSearch/pull/6129)) -- Bumps `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130)) ### Changed -- Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725)) -- [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947)) -- Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008)) -- [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976)) -- Update API spec for cluster health API ([#6399](https://github.com/opensearch-project/OpenSearch/pull/6399)) +- Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009)) - [Refactor] XContent base classes from xcontent to core library ([#5902](https://github.com/opensearch-project/OpenSearch/pull/5902)) ### Deprecated @@ -125,11 +82,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Removed ### Fixed -- [Segment Replication] Fix for peer recovery ([#5344](https://github.com/opensearch-project/OpenSearch/pull/5344)) -- Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004)) -- [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122)) -- Enable numeric sort optimisation for few numerical sort types ([#6321](https://github.com/opensearch-project/OpenSearch/pull/6321)) -- Fix Opensearch repository-s3 plugin cannot read ServiceAccount token ([#6390](https://github.com/opensearch-project/OpenSearch/pull/6390) ### Security diff --git a/release-notes/opensearch.release-notes-2.6.0.md b/release-notes/opensearch.release-notes-2.6.0.md new file mode 100644 index 0000000000000..860e789be06ac --- /dev/null +++ b/release-notes/opensearch.release-notes-2.6.0.md @@ -0,0 +1,57 @@ +## 2023-02-22 Version 2.6.0 Release Notes + +## [2.6] +### Added +- Add index create block when all nodes have breached high disk watermark ([#5852](https://github.com/opensearch-project/OpenSearch/pull/5852)) +- Add cluster manager throttling stats in nodes/stats API ([#5790](https://github.com/opensearch-project/OpenSearch/pull/5790)) +- Add support for feature flags in opensearch.yml ([#4959](https://github.com/opensearch-project/OpenSearch/pull/4959)) +- Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658)) +- Add update-index-settings allowlist for searchable snapshot ([#5907](https://github.com/opensearch-project/OpenSearch/pull/5907)) +- Add new cat/segment_replication API to surface Segment Replication metrics ([#5718](https://github.com/opensearch-project/OpenSearch/pull/5718)). +- Replace latches with CompletableFutures for extensions ([#5646](https://github.com/opensearch-project/OpenSearch/pull/5646)) +- Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874)) +- Add support to apply index create block ([#4603](https://github.com/opensearch-project/OpenSearch/issues/4603)) +- Add support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003)) +- Add a guardrail to limit maximum number of shard on the cluster ([#6143](https://github.com/opensearch-project/OpenSearch/pull/6143)) +- Add cancellation of in-flight SearchTasks based on resource consumption ([#5606](https://github.com/opensearch-project/OpenSearch/pull/5605)) +- Add support for ppc64le architecture ([#5459](https://github.com/opensearch-project/OpenSearch/pull/5459)) +- Add a setting to control auto release of OpenSearch managed index creation block ([#6277](https://github.com/opensearch-project/OpenSearch/pull/6277)) +- Fix timeout error when adding a document to an index with extension running ([#6275](https://github.com/opensearch-project/OpenSearch/pull/6275)) +- Handle translog upload during primary relocation for remote-backed indexes ([#5804](https://github.com/opensearch-project/OpenSearch/pull/5804)) +- Batch translog sync/upload per x ms for remote-backed indexes ([#5854](https://github.com/opensearch-project/OpenSearch/pull/5854)) + +### Dependencies +- Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704)) +- Bump `reactor-netty` from 1.1.1 to 1.1.2 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/5878)) +- OpenJDK Update (January 2023 Patch releases) ([#6075](https://github.com/opensearch-project/OpenSearch/pull/6075)) +- Bump `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6088](https://github.com/opensearch-project/OpenSearch/pull/6088)) +- Bump `joda` from 2.10.13 to 2.12.2 ([#6095](https://github.com/opensearch-project/OpenSearch/pull/6095)) +- Upgrade to Lucene 9.5.0 ([#6078](https://github.com/opensearch-project/OpenSearch/pull/6078)) +- Bump antlr4 from 4.9.3 to 4.11.1 ([#6116](https://github.com/opensearch-project/OpenSearch/pull/6116)) +- Bump `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130)) +- Bump `Jackson` from 2.14.1 to 2.14.2 ([#6129](https://github.com/opensearch-project/OpenSearch/pull/6129)) +- Bump `org.apache.ant:ant` from 1.10.12 to 1.10.13 ([#6306](https://github.com/opensearch-project/OpenSearch/pull/6306)) +- Bump `azure-core-http-netty` from 1.12.7 to 1.12.8 ([#6304](https://github.com/opensearch-project/OpenSearch/pull/6304)) +- Bump `com.azure:azure-storage-common` from 12.18.1 to 12.19.3 ([#6304](https://github.com/opensearch-project/OpenSearch/pull/6304)) +- Bump `reactor-netty-http` from 1.0.24 to 1.1.2 ([#6309](https://github.com/opensearch-project/OpenSearch/pull/6309)) +- Bump `com.google.protobuf:protobuf-java` from 3.21.12 to 3.22.0 ([#6387](https://github.com/opensearch-project/OpenSearch/pull/6387)) +- Bump `io.projectreactor.netty:reactor-netty` from 1.1.2 to 1.1.3 ([#6386](https://github.com/opensearch-project/OpenSearch/pull/6386)) +- Bump `com.diffplug.spotless` from 6.10.0 to 6.15.0 ([#6385](https://github.com/opensearch-project/OpenSearch/pull/6385)) +- Bump `jettison` from 1.5.1 to 1.5.3 ([#6391](https://github.com/opensearch-project/OpenSearch/pull/6391)) + +### Changed +- Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725)) +- [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947)) +- Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008)) +- [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976)) +- Update API spec for cluster health API ([#6399](https://github.com/opensearch-project/OpenSearch/pull/6399)) + +### Removed +- Remove deprecated org.gradle.util.DistributionLocator usage ([#6212](https://github.com/opensearch-project/OpenSearch/pull/6212)) + +### Fixed +- [Segment Replication] Fix for peer recovery ([#5344](https://github.com/opensearch-project/OpenSearch/pull/5344)) +- Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004)) +- [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122)) +- Enable numeric sort optimisation for few numerical sort types ([#6321](https://github.com/opensearch-project/OpenSearch/pull/6321)) +- Fix Opensearch repository-s3 plugin cannot read ServiceAccount token ([#6390](https://github.com/opensearch-project/OpenSearch/pull/6390)