Merge branch 'main' into decommission-api/pr

CHANGELOG.md

@@ -38,6 +38,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
+- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4 ([#4644](https://github.com/opensearch-project/OpenSearch/pull/4644))
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
@@ -53,9 +54,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084))
 - Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360))
 - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
+- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582))
 - Add APIs (GET/PUT) to decommission awareness attribute ([#4261](https://github.com/opensearch-project/OpenSearch/pull/4261))
 
-
 ### Deprecated
 
 ### Removed
@@ -88,6 +89,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544))
 - Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
 - [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))
+- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

plugins/repository-hdfs/build.gradle

@@ -48,7 +48,7 @@ opensearchplugin {
 }
 
 versions << [
-  'hadoop3': '3.3.3'
+  'hadoop3': '3.3.4'
 ]
 
 testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"

plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1

@@ -0,0 +1 @@
+6339a8f7279310c8b1f7ef314b592d8c71ca72ef

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1

@@ -0,0 +1 @@
+21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1

@@ -0,0 +1 @@
+036ef2f86dc44410d2bb5d54ce40435d2484d9a5

server/src/main/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverService.java

@@ -188,7 +188,7 @@ private RolloverResult rolloverAlias(
         ClusterState newState = createIndexService.applyCreateIndexRequest(currentState, createIndexClusterStateRequest, silent);
         newState = indexAliasesService.applyAliasActions(
             newState,
-            rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata.isHidden(), aliasName)
+            rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata, aliasName)
         );
 
         RolloverInfo rolloverInfo = new RolloverInfo(aliasName, metConditions, threadPool.absoluteTimeInMillis());
@@ -309,20 +309,46 @@ static List<AliasAction> rolloverAliasToNewIndex(
         String oldIndex,
         String newIndex,
         boolean explicitWriteIndex,
-        @Nullable Boolean isHidden,
+        AliasMetadata aliasMetadata,
         String alias
     ) {
+        String filterAsString = aliasMetadata.getFilter() != null ? aliasMetadata.getFilter().string() : null;
+
         if (explicitWriteIndex) {
             return Collections.unmodifiableList(
                 Arrays.asList(
-                    new AliasAction.Add(newIndex, alias, null, null, null, true, isHidden),
-                    new AliasAction.Add(oldIndex, alias, null, null, null, false, isHidden)
+                    new AliasAction.Add(
+                        newIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        true,
+                        aliasMetadata.isHidden()
+                    ),
+                    new AliasAction.Add(
+                        oldIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        false,
+                        aliasMetadata.isHidden()
+                    )
                 )
             );
         } else {
             return Collections.unmodifiableList(
                 Arrays.asList(
-                    new AliasAction.Add(newIndex, alias, null, null, null, null, isHidden),
+                    new AliasAction.Add(
+                        newIndex,
+                        alias,
+                        filterAsString,
+                        aliasMetadata.getIndexRouting(),
+                        aliasMetadata.getSearchRouting(),
+                        null,
+                        aliasMetadata.isHidden()
+                    ),
                     new AliasAction.Remove(oldIndex, alias, null)
                 )
             );

server/src/main/java/org/opensearch/cluster/metadata/AliasAction.java

@@ -138,6 +138,18 @@ public String getAlias() {
             return alias;
         }
 
+        public String getFilter() {
+            return filter;
+        }
+
+        public String getSearchRouting() {
+            return searchRouting;
+        }
+
+        public String getIndexRouting() {
+            return indexRouting;
+        }
+
         public Boolean writeIndex() {
             return writeIndex;
         }

server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

@@ -616,11 +616,18 @@ public static void setAdditionalRoles(final Set<DiscoveryNodeRole> additionalRol
             + "], roles by name abbreviation ["
             + roleNameAbbreviationToPossibleRoles
             + "]";
-        // TODO: Remove the Map 'roleNameToPossibleRolesWithMaster' and let 'roleMap = roleNameToPossibleRoles', after removing MASTER_ROLE.
-        // It's used to allow CLUSTER_MANAGER_ROLE that introduced in 2.0, having the same abbreviation name with MASTER_ROLE.
-        final Map<String, DiscoveryNodeRole> roleNameToPossibleRolesWithMaster = new HashMap<>(roleNameToPossibleRoles);
-        roleNameToPossibleRolesWithMaster.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
-        roleMap = Collections.unmodifiableMap(roleNameToPossibleRolesWithMaster);
+        roleMap = roleNameToPossibleRoles;
+    }
+
+    /**
+     * Load the deprecated {@link DiscoveryNodeRole#MASTER_ROLE}.
+     * Master role is not added into BUILT_IN_ROLES, because {@link #setAdditionalRoles(Set)} check role name abbreviation duplication,
+     * and CLUSTER_MANAGER_ROLE has the same abbreviation name with MASTER_ROLE.
+     */
+    public static void setDeprecatedMasterRole() {
+        final Map<String, DiscoveryNodeRole> modifiableRoleMap = new HashMap<>(roleMap);
+        modifiableRoleMap.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
+        roleMap = Collections.unmodifiableMap(modifiableRoleMap);
     }
 
     public static Set<String> getPossibleRoleNames() {

server/src/main/java/org/opensearch/node/Node.java

@@ -429,6 +429,8 @@ protected Node(
                 .collect(Collectors.toSet());
             DiscoveryNode.setAdditionalRoles(additionalRoles);
 
+            DiscoveryNode.setDeprecatedMasterRole();
+
             /*
              * Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting
              * values, no matter they ask for them from.

server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -126,7 +126,13 @@ public void testRolloverAliasActions() {
         String sourceIndex = randomAlphaOfLength(10);
         String targetIndex = randomAlphaOfLength(10);
 
-        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, false, null, sourceAlias);
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            false,
+            createDefaultAliasMetadata(sourceAlias, null),
+            sourceAlias
+        );
         assertThat(actions, hasSize(2));
         boolean foundAdd = false;
         boolean foundRemove = false;
@@ -149,7 +155,13 @@ public void testRolloverAliasActionsWithExplicitWriteIndex() {
         String sourceAlias = randomAlphaOfLength(10);
         String sourceIndex = randomAlphaOfLength(10);
         String targetIndex = randomAlphaOfLength(10);
-        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, true, null, sourceAlias);
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            true,
+            createDefaultAliasMetadata(sourceAlias, null),
+            sourceAlias
+        );
 
         assertThat(actions, hasSize(2));
         boolean foundAddWrite = false;
@@ -172,11 +184,64 @@ public void testRolloverAliasActionsWithExplicitWriteIndex() {
         assertTrue(foundRemoveWrite);
     }
 
+    public void testRolloverAliasActionsWithFilterAndExplicitWriteIndex() {
+        String sourceAlias = randomAlphaOfLength(10);
+        String sourceIndex = randomAlphaOfLength(10);
+        String targetIndex = randomAlphaOfLength(10);
+        String indexRouting = randomAlphaOfLength(10);
+        String sourceRouting = randomAlphaOfLength(10);
+        AliasMetadata aliasMetadata = createAliasMetadata(
+            sourceAlias,
+            Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)),
+            indexRouting,
+            sourceRouting,
+            true
+        );
+
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            true,
+            aliasMetadata,
+            sourceAlias
+        );
+
+        assertThat(actions, hasSize(2));
+        boolean foundAddWrite = false;
+        boolean foundRemoveWrite = false;
+        for (AliasAction action : actions) {
+            AliasAction.Add addAction = (AliasAction.Add) action;
+            if (action.getIndex().equals(targetIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertEquals(aliasMetadata.filter().string(), addAction.getFilter());
+                assertEquals(indexRouting, addAction.getIndexRouting());
+                assertEquals(sourceRouting, addAction.getSearchRouting());
+
+                assertTrue(addAction.writeIndex());
+                foundAddWrite = true;
+            } else if (action.getIndex().equals(sourceIndex)) {
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertFalse(addAction.writeIndex());
+                foundRemoveWrite = true;
+            } else {
+                throw new AssertionError("Unknown index [" + action.getIndex() + "]");
+            }
+        }
+        assertTrue(foundAddWrite);
+        assertTrue(foundRemoveWrite);
+    }
+
     public void testRolloverAliasActionsWithHiddenAliasAndExplicitWriteIndex() {
         String sourceAlias = randomAlphaOfLength(10);
         String sourceIndex = randomAlphaOfLength(10);
         String targetIndex = randomAlphaOfLength(10);
-        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, true, true, sourceAlias);
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            true,
+            createDefaultAliasMetadata(sourceAlias, true),
+            sourceAlias
+        );
 
         assertThat(actions, hasSize(2));
         boolean foundAddWrite = false;
@@ -202,11 +267,66 @@ public void testRolloverAliasActionsWithHiddenAliasAndExplicitWriteIndex() {
         assertTrue(foundRemoveWrite);
     }
 
+    public void testRolloverAliasActionsWithFilterAndHiddenAliasAndImplicitWriteIndex() {
+        String sourceAlias = randomAlphaOfLength(10);
+        String sourceIndex = randomAlphaOfLength(10);
+        String targetIndex = randomAlphaOfLength(10);
+        String indexRouting = randomAlphaOfLength(10);
+        String sourceRouting = randomAlphaOfLength(10);
+        AliasMetadata aliasMetadata = createAliasMetadata(
+            sourceAlias,
+            Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)),
+            indexRouting,
+            sourceRouting,
+            true
+        );
+
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            false,
+            aliasMetadata,
+            sourceAlias
+        );
+
+        assertThat(actions, hasSize(2));
+        boolean foundAddWrite = false;
+        boolean foundRemoveWrite = false;
+        for (AliasAction action : actions) {
+            if (action.getIndex().equals(targetIndex)) {
+                assertThat(action, instanceOf(AliasAction.Add.class));
+                AliasAction.Add addAction = (AliasAction.Add) action;
+                assertEquals(sourceAlias, addAction.getAlias());
+                assertThat(addAction.writeIndex(), nullValue());
+                assertTrue(addAction.isHidden());
+                assertEquals(aliasMetadata.filter().string(), addAction.getFilter());
+                assertEquals(indexRouting, addAction.getIndexRouting());
+                assertEquals(sourceRouting, addAction.getSearchRouting());
+                foundAddWrite = true;
+            } else if (action.getIndex().equals(sourceIndex)) {
+                assertThat(action, instanceOf(AliasAction.Remove.class));
+                AliasAction.Remove removeAction = (AliasAction.Remove) action;
+                assertEquals(sourceAlias, removeAction.getAlias());
+                foundRemoveWrite = true;
+            } else {
+                throw new AssertionError("Unknown index [" + action.getIndex() + "]");
+            }
+        }
+        assertTrue(foundAddWrite);
+        assertTrue(foundRemoveWrite);
+    }
+
     public void testRolloverAliasActionsWithHiddenAliasAndImplicitWriteIndex() {
         String sourceAlias = randomAlphaOfLength(10);
         String sourceIndex = randomAlphaOfLength(10);
         String targetIndex = randomAlphaOfLength(10);
-        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, false, true, sourceAlias);
+        List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
+            sourceIndex,
+            targetIndex,
+            false,
+            createDefaultAliasMetadata(sourceAlias, true),
+            sourceAlias
+        );
 
         assertThat(actions, hasSize(2));
         boolean foundAddWrite = false;
@@ -1010,4 +1130,23 @@ private static IndexMetadata createMetadata(String indexName) {
             .settings(settings)
             .build();
     }
+
+    private static AliasMetadata createDefaultAliasMetadata(String alias, Boolean isHidden) {
+        return AliasMetadata.builder(alias).isHidden(isHidden).build();
+    }
+
+    private static AliasMetadata createAliasMetadata(
+        String alias,
+        Map filter,
+        String indexRouting,
+        String searchRouting,
+        Boolean isHidden
+    ) {
+        return AliasMetadata.builder(alias)
+            .isHidden(isHidden)
+            .filter(filter)
+            .indexRouting(indexRouting)
+            .searchRouting(searchRouting)
+            .build();
+    }
 }

server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeRoleSettingTests.java

@@ -56,8 +56,7 @@ public void testIsIngestNode() {
     }
 
     public void testIsMasterNode() {
-        // It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
-        DiscoveryNode.setAdditionalRoles(Collections.emptySet());
+        DiscoveryNode.setDeprecatedMasterRole();
         runRoleTest(DiscoveryNode::isClusterManagerNode, DiscoveryNodeRole.MASTER_ROLE);
     }
 

server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java

@@ -176,11 +176,11 @@ public void testDiscoveryNodeIsRemoteClusterClientUnset() {
     }
 
     // Added in 2.0 temporarily, validate the MASTER_ROLE is in the list of known roles.
-    // MASTER_ROLE was removed from BUILT_IN_ROLES and is imported by setAdditionalRoles(),
+    // MASTER_ROLE was removed from BUILT_IN_ROLES and is imported by setDeprecatedMasterRole(),
     // as a workaround for making the new CLUSTER_MANAGER_ROLE has got the same abbreviation 'm'.
     // The test validate this behavior.
-    public void testSetAdditionalRolesCanAddDeprecatedMasterRole() {
-        DiscoveryNode.setAdditionalRoles(Collections.emptySet());
+    public void testSetDeprecatedMasterRoleCanAddMasterRole() {
+        DiscoveryNode.setDeprecatedMasterRole();
         assertTrue(DiscoveryNode.getPossibleRoleNames().contains(DiscoveryNodeRole.MASTER_ROLE.roleName()));
     }
 

server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java

@@ -26,8 +26,7 @@ public class NodeRoleSettingsTests extends OpenSearchTestCase {
      * Remove the test after removing MASTER_ROLE.
      */
     public void testClusterManagerAndMasterRoleCanNotCoexist() {
-        // It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
-        DiscoveryNode.setAdditionalRoles(Collections.emptySet());
+        DiscoveryNode.setDeprecatedMasterRole();
         Settings roleSettings = Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "cluster_manager, master").build();
         Exception exception = expectThrows(IllegalArgumentException.class, () -> NodeRoleSettings.NODE_ROLES_SETTING.get(roleSettings));
         assertThat(exception.getMessage(), containsString("[master, cluster_manager] can not be assigned together to a node"));
@@ -49,8 +48,7 @@ public void testClusterManagerAndDataNodeRoles() {
      * Remove the test after removing MASTER_ROLE.
      */
     public void testMasterRoleDeprecationMessage() {
-        // It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
-        DiscoveryNode.setAdditionalRoles(Collections.emptySet());
+        DiscoveryNode.setDeprecatedMasterRole();
         Settings roleSettings = Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master").build();
         assertEquals(Collections.singletonList(DiscoveryNodeRole.MASTER_ROLE), NodeRoleSettings.NODE_ROLES_SETTING.get(roleSettings));
         assertWarnings(DiscoveryNodeRole.MASTER_ROLE_DEPRECATION_MESSAGE);