Initial support for opensearch 2.x (#126)

* Initial support for opensearch 2.x Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com> * Fix some lingering tests Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com> --------- Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
opensearch-project · Feb 25, 2023 · f4198b5 · f4198b5
1 parent e40d44a
commit f4198b5
Show file tree

Hide file tree

Showing 17 changed files with 78 additions and 132 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,10 @@
 # CHANGELOG
 Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
-## [Unreleased]
+## [Unreleased 2.x]
 ### Added
 - Added CHANGELOG and verifier workflow ([65](https://github.com/opensearch-project/opensearch-hadoop/pull/65))
+- Initial support for OpenSearch 2.x ([126](https://github.com/opensearch-project/opensearch-hadoop/pull/126))
 ### Changed
 - [Spark Distribution] Default Assemble artifact to Spark 3 ([107](https://github.com/opensearch-project/opensearch-hadoop/pull/107))
 ### Deprecated
@@ -28,4 +29,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `com.amazonaws:aws-java-sdk-bundle` from 1.12.397 to 1.12.411
 
 
-[Unreleased]: https://github.com/opensearch-project/opensearch-hadoop/compare/main...HEAD
+[Unreleased 2.x]: https://github.com/opensearch-project/opensearch-hadoop/compare/2.0...HEAD
diff --git a/buildSrc/opensearch-hadoop-version.properties b/buildSrc/opensearch-hadoop-version.properties
@@ -1,4 +1,4 @@
-opensearch_hadoop        = 3.0.0
-opensearch      = 3.0.0
-lucene          = 9.5.0-snapshot-a4ef70f
+opensearch_hadoop        = 2.0.0
+opensearch      = 2.6.0
+lucene          = 9.5.0
 build-tools     = 3.0.0
diff --git a/mr/src/main/java/org/opensearch/hadoop/util/OpenSearchMajorVersion.java b/mr/src/main/java/org/opensearch/hadoop/util/OpenSearchMajorVersion.java
@@ -39,9 +39,9 @@ public class OpenSearchMajorVersion implements Serializable {
 
     private static final long serialVersionUID = 1L;
 
+    public static final OpenSearchMajorVersion V_1_X = new OpenSearchMajorVersion((byte) 1, "1.x");
     public static final OpenSearchMajorVersion V_2_X = new OpenSearchMajorVersion((byte) 2, "2.x");
-    public static final OpenSearchMajorVersion V_3_X = new OpenSearchMajorVersion((byte) 3, "3.x");
-    public static final OpenSearchMajorVersion LATEST = V_3_X;
+    public static final OpenSearchMajorVersion LATEST = V_2_X;
 
     public final byte major;
     private final String version;
@@ -76,18 +76,12 @@ public boolean onOrBefore(OpenSearchMajorVersion version) {
     }
 
     public static OpenSearchMajorVersion parse(String version) {
-        if (version.startsWith("0.")) {
-            return new OpenSearchMajorVersion((byte) 0, version);
-        }
         if (version.startsWith("1.")) {
             return new OpenSearchMajorVersion((byte) 1, version);
         }
         if (version.startsWith("2.")) {
             return new OpenSearchMajorVersion((byte) 2, version);
         }
-        if (version.startsWith("3.")) {
-            return new OpenSearchMajorVersion((byte) 3, version);
-        }
         throw new OpenSearchHadoopIllegalArgumentException("Unsupported/Unknown OpenSearch version [" + version + "]." +
                 "Highest supported version is [" + LATEST.version + "]. You may need to upgrade OpenSearch-Hadoop.");
     }

diff --git a/mr/src/test/java/org/opensearch/hadoop/rest/ApplyAliasMetaDataTest.java b/mr/src/test/java/org/opensearch/hadoop/rest/ApplyAliasMetaDataTest.java
@@ -54,7 +54,7 @@ public class ApplyAliasMetaDataTest {
 
     private static final OpenSearchMajorVersion[] OPENSEARCH_VERSIONS =
             new OpenSearchMajorVersion[]{
-                    OpenSearchMajorVersion.V_3_X
+                    OpenSearchMajorVersion.V_2_X
             };
 
     @Test

diff --git a/mr/src/test/java/org/opensearch/hadoop/rest/InitializationUtilsTest.java b/mr/src/test/java/org/opensearch/hadoop/rest/InitializationUtilsTest.java
@@ -134,15 +134,15 @@ public void testValidateMultipleScripts() throws Exception {
     @Test(expected = OpenSearchHadoopIllegalArgumentException.class)
     public void testValidateWriteTTLRemoved() throws Exception {
         Settings set = new TestSettings();
-        set.setInternalClusterInfo(ClusterInfo.unnamedClusterWithVersion(OpenSearchMajorVersion.V_3_X));
+        set.setInternalClusterInfo(ClusterInfo.unnamedClusterWithVersion(OpenSearchMajorVersion.V_2_X));
         set.setProperty(OPENSEARCH_MAPPING_TTL, "1000");
         validateSettingsForWriting(set);
     }
 
     @Test(expected = OpenSearchHadoopIllegalArgumentException.class)
     public void testValidateWriteTimestampRemoved() throws Exception {
         Settings set = new TestSettings();
-        set.setInternalClusterInfo(ClusterInfo.unnamedClusterWithVersion(OpenSearchMajorVersion.V_3_X));
+        set.setInternalClusterInfo(ClusterInfo.unnamedClusterWithVersion(OpenSearchMajorVersion.V_2_X));
         set.setProperty(OPENSEARCH_MAPPING_TIMESTAMP, "1000");
         validateSettingsForWriting(set);
     }

diff --git a/mr/src/test/java/org/opensearch/hadoop/rest/ResourceTest.java b/mr/src/test/java/org/opensearch/hadoop/rest/ResourceTest.java
@@ -54,11 +54,9 @@ public static Collection<Object[]> params() {
         parameters.add(new Object[]{OpenSearchMajorVersion.LATEST, true});
         parameters.add(new Object[]{OpenSearchMajorVersion.LATEST, false});
 
-        parameters.add(new Object[]{OpenSearchMajorVersion.V_3_X, true});
-        parameters.add(new Object[]{OpenSearchMajorVersion.V_2_X, true});
+        parameters.add(new Object[]{OpenSearchMajorVersion.V_1_X, true});
+        parameters.add(new Object[]{OpenSearchMajorVersion.V_1_X, false});
 
-        parameters.add(new Object[]{OpenSearchMajorVersion.V_3_X, false});
-        parameters.add(new Object[]{OpenSearchMajorVersion.V_2_X, false});
         return parameters;
     }
 

diff --git a/mr/src/test/java/org/opensearch/hadoop/rest/RestClientTest.java b/mr/src/test/java/org/opensearch/hadoop/rest/RestClientTest.java
@@ -201,7 +201,7 @@ public void testCount() throws Exception {
                 .thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200"));
 
         Settings testSettings = new TestSettings();
-        testSettings.setInternalVersion(OpenSearchMajorVersion.V_3_X);
+        testSettings.setInternalVersion(OpenSearchMajorVersion.V_2_X);
         RestClient client = new RestClient(testSettings, mock);
 
         // Make sure that it works

diff --git a/mr/src/test/java/org/opensearch/hadoop/rest/bulk/BulkProcessorTest.java b/mr/src/test/java/org/opensearch/hadoop/rest/bulk/BulkProcessorTest.java
@@ -86,7 +86,7 @@ public class BulkProcessorTest {
     public static Collection<Object[]> data() {
         List<Object[]> params = new ArrayList<Object[]>();
 
-        params.add(new Object[]{OpenSearchMajorVersion.V_3_X, new BulkOutputGeneratorV3()});
+        params.add(new Object[]{OpenSearchMajorVersion.V_2_X, new BulkOutputGeneratorV3()});
 
         return params;
     }

diff --git a/mr/src/test/java/org/opensearch/hadoop/serialization/CommandTest.java b/mr/src/test/java/org/opensearch/hadoop/serialization/CommandTest.java
@@ -69,7 +69,7 @@ public class CommandTest {
     public static Collection<Object[]> data() {
 
         // make sure all versions are tested. Throw if a new one is seen:
-        if (OpenSearchMajorVersion.LATEST != OpenSearchMajorVersion.V_3_X) {
+        if (OpenSearchMajorVersion.LATEST != OpenSearchMajorVersion.V_2_X) {
             throw new IllegalStateException("CommandTest needs new version updates.");
         }
 
@@ -81,8 +81,7 @@ public static Collection<Object[]> data() {
                 ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT,
                 ConfigurationOptions.OPENSEARCH_OPERATION_DELETE};
         boolean[] asJsons = new boolean[]{false, true};
-        OpenSearchMajorVersion[] versions = new OpenSearchMajorVersion[]{OpenSearchMajorVersion.V_2_X,
-                OpenSearchMajorVersion.V_3_X};
+        OpenSearchMajorVersion[] versions = new OpenSearchMajorVersion[] { OpenSearchMajorVersion.V_2_X };
 
         for (OpenSearchMajorVersion version : versions) {
             for (boolean asJson : asJsons) {
@@ -141,7 +140,7 @@ public void testConstantId() throws Exception {
 
     @Test
     public void testParent() throws Exception {
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         assumeFalse(ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT.equals(operation));
         assumeFalse(isDeleteOP() && jsonInput);
         Settings settings = settings();
@@ -154,7 +153,7 @@ public void testParent() throws Exception {
 
     @Test
     public void testVersion() throws Exception {
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         assumeFalse(ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT.equals(operation));
         assumeFalse(isDeleteOP() && jsonInput);
         Settings settings = settings();
@@ -191,7 +190,7 @@ public void testTimestamp() throws Exception {
 
     @Test
     public void testRouting() throws Exception {
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         assumeFalse(ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT.equals(operation));
         assumeFalse(isDeleteOP() && jsonInput);
         Settings settings = settings();
@@ -204,7 +203,7 @@ public void testRouting() throws Exception {
 
     @Test
     public void testAllX() throws Exception {
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         assumeFalse(ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT.equals(operation));
         assumeFalse(isDeleteOP() && jsonInput);
         Settings settings = settings();
@@ -250,7 +249,7 @@ public void testIdMandatory() throws Exception {
     @Test
     public void testUpdateOnlyInlineScript() throws Exception {
         assumeTrue(ConfigurationOptions.OPENSEARCH_OPERATION_UPDATE.equals(operation));
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         Settings set = settings();
 
         set.setProperty(ConfigurationOptions.OPENSEARCH_INDEX_AUTO_CREATE, "yes");
@@ -268,7 +267,7 @@ public void testUpdateOnlyInlineScript() throws Exception {
     @Test
     public void testUpdateOnlyFileScript() throws Exception {
         assumeTrue(ConfigurationOptions.OPENSEARCH_OPERATION_UPDATE.equals(operation));
-        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_3_X));
+        assumeTrue(version.onOrAfter(OpenSearchMajorVersion.V_2_X));
         Settings set = settings();
 
         set.setProperty(ConfigurationOptions.OPENSEARCH_INDEX_AUTO_CREATE, "yes");

diff --git a/mr/src/test/java/org/opensearch/hadoop/serialization/dto/NodeInfoTest.java b/mr/src/test/java/org/opensearch/hadoop/serialization/dto/NodeInfoTest.java
@@ -50,13 +50,6 @@ public void testV2() throws Exception {
         assertFalse(nodeMap.get("Unseen").isIngest());
     }
 
-    @Test
-    public void testV5() throws Exception {
-        Map<String, NodeInfo> nodeMap = testNodeInfo(getClass().getResourceAsStream("client-nodes-v5.json"));
-        assertFalse(nodeMap.get("Darkhawk").isIngest());
-        assertTrue(nodeMap.get("Unseen").isIngest());
-    }
-
     static Map<String, NodeInfo> testNodeInfo(InputStream input) throws IOException {
         ObjectMapper mapper = new ObjectMapper();
         JsonParser jsonParser = mapper.getJsonFactory().createJsonParser(input);

diff --git a/mr/src/test/java/org/opensearch/hadoop/serialization/field/DefaultIndexExtractorTest.java b/mr/src/test/java/org/opensearch/hadoop/serialization/field/DefaultIndexExtractorTest.java
@@ -57,7 +57,7 @@ public class DefaultIndexExtractorTest {
     public void createFieldExtractor() {
         Settings settings = new TestSettings();
         settings.setResourceWrite("{field}");
-        settings.setInternalVersion(OpenSearchMajorVersion.V_3_X);
+        settings.setInternalVersion(OpenSearchMajorVersion.V_2_X);
         InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG);
 
         IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings);
@@ -76,7 +76,7 @@ public void createFieldExtractor() {
     public void createFieldExtractorNull() {
         Settings settings = new TestSettings();
         settings.setResourceWrite("test/{field}");
-        settings.setInternalVersion(OpenSearchMajorVersion.V_3_X);
+        settings.setInternalVersion(OpenSearchMajorVersion.V_2_X);
         InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG);
 
         IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings);

diff --git a/mr/src/test/java/org/opensearch/hadoop/util/OpenSearchMajorVersionTest.java b/mr/src/test/java/org/opensearch/hadoop/util/OpenSearchMajorVersionTest.java
@@ -78,10 +78,8 @@ public class OpenSearchMajorVersionTest {
         versions.add("2.3.0");
         versions.add("2.3.1");
         versions.add("2.4.0");
-        versions.add("3.0.0-alpha1");
-        versions.add("3.0.0-beta2");
-        versions.add("3.0.0-rc3");
-        versions.add("3.0.0");
+        versions.add("2.5.0");
+        versions.add("2.6.0");
         TEST_VERSIONS = Collections.unmodifiableList(versions);
     }
 
@@ -119,24 +117,24 @@ public void testMinorVersionParsing() {
             assertThat(minorVersion, greaterThanOrEqualTo(0));
         }
         try {
-            OpenSearchMajorVersion.V_3_X.parseMinorVersion("6.0.0");
+            OpenSearchMajorVersion.V_2_X.parseMinorVersion("6.0.0");
             fail("Invalid major version");
         } catch (OpenSearchHadoopIllegalArgumentException e) {
-            assertEquals("Invalid version string for major version; Received [6.0.0] for major version [3.x]",
+            assertEquals("Invalid version string for major version; Received [6.0.0] for major version [2.x]",
                     e.getMessage());
         }
         try {
-            OpenSearchMajorVersion.V_3_X.parseMinorVersion("3.");
+            OpenSearchMajorVersion.V_2_X.parseMinorVersion("2.");
             fail("Invalid major version");
         } catch (OpenSearchHadoopIllegalArgumentException e) {
-            assertEquals("Could not parse OpenSearch minor version [3.]. Invalid version format.",
+            assertEquals("Could not parse OpenSearch minor version [2.]. Invalid version format.",
                     e.getMessage());
         }
         try {
-            OpenSearchMajorVersion.V_3_X.parseMinorVersion("3.4-abcd.4");
+            OpenSearchMajorVersion.V_2_X.parseMinorVersion("2.6-abcd.4");
             fail("Invalid major version");
         } catch (OpenSearchHadoopIllegalArgumentException e) {
-            assertEquals("Could not parse OpenSearch minor version [3.4-abcd.4]. Non-numeric minor version [4-abcd].",
+            assertEquals("Could not parse OpenSearch minor version [2.6-abcd.4]. Non-numeric minor version [6-abcd].",
                     e.getMessage());
         }
     }

diff --git a/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v1.json b/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v1.json
@@ -1,36 +1,35 @@
 {
-    "ok" : true,
-    "cluster_name" : "opensearch",
-    "nodes" : {
-        "fiR5azTbTDiX59m78yzOTw" : {
-            "name" : "Darkhawk",
-            "transport_address" : "inet[/192.168.1.50:9300]",
-            "host" : "cerberus",
-            "version" : "1.7.3",
-            "http_address" : "inet[/192.168.1.50:9200]",
-            "attributes" : {
-                "data" : "false",
-                "master" : "false"
-            },
-            "http" : {
-                "bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]",
-                "publish_address" : "inet[/192.168.1.50:9200]",
-                "max_content_length" : "100mb",
-                "max_content_length_in_bytes" : 104857600
-            }
-        },
-        "L9DM79IvStq2RStogXR8Sg" : {
-            "name" : "Unseen",
-            "transport_address" : "inet[/192.168.1.50:9301]",
-            "host" : "cerberus",
-            "version" : "1.7.3",
-            "http_address" : "inet[/192.168.1.50:9201]",
-            "http" : {
-                "bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9201]",
-                "publish_address" : "inet[/192.168.1.50:9201]",
-                "max_content_length" : "100mb",
-                "max_content_length_in_bytes" : 104857600
-            }
-        }
+{
+  "ok": true,
+  "cluster_name": "opensearch",
+  "nodes": {
+    "fiR5azTbTDiX59m78yzOTw": {
+      "name": "Darkhawk",
+      "transport_address": "inet[/192.168.1.50:9300]",
+      "host": "cerberus",
+      "version": "1.3.8",
+      "roles": [],
+      "http": {
+        "bound_address": "inet[/0:0:0:0:0:0:0:0:9200]",
+        "publish_address": "inet[/192.168.1.50:9200]",
+        "max_content_length": "100mb",
+        "max_content_length_in_bytes": 104857600
+      }
+    },
+    "L9DM79IvStq2RStogXR8Sg": {
+      "name": "Unseen",
+      "transport_address": "inet[/192.168.1.50:9301]",
+      "host": "cerberus",
+      "version": "1.3.8",
+      "roles": ["master", "data"],
+      "http_address": "inet[/192.168.1.50:9201]",
+      "http": {
+        "bound_address": "inet[/0:0:0:0:0:0:0:0%0:9201]",
+        "publish_address": "inet[/192.168.1.50:9201]",
+        "max_content_length": "100mb",
+        "max_content_length_in_bytes": 104857600
+      }
     }
+  }
+}
 }
diff --git a/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v2.json b/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v2.json
@@ -6,7 +6,7 @@
             "name" : "Darkhawk",
             "transport_address" : "inet[/192.168.1.50:9300]",
             "host" : "cerberus",
-            "version" : "2.3.3",
+            "version" : "2.5.0",
             "roles" : [],
             "http" : {
                 "bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]",
@@ -19,9 +19,9 @@
             "name" : "Unseen",
             "transport_address" : "inet[/192.168.1.50:9301]",
             "host" : "cerberus",
-            "version" : "3.3.3",
+            "version" : "2.6.0",
             "roles" : [
-                "master",
+                "cluster_manager",
                 "data"
             ],
             "http_address" : "inet[/192.168.1.50:9201]",

diff --git a/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v5.json b/mr/src/test/resources/org/opensearch/hadoop/serialization/dto/client-nodes-v5.json