[Bug] Fix mixed cluster support for OpenSearch 2+ (#1191)

The version framework only added support for OpenSearch 1.x bwc with legacy
clusters. This commit adds support for v2.0 which will be the last version with
bwc support for legacy clusters (v7.10)

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

server/src/main/java/org/opensearch/LegacyESVersion.java

@@ -167,7 +167,7 @@ public class LegacyESVersion extends Version {
                             final int minor = Integer.valueOf(fields[2]) * 10000;
                             final int revision = Integer.valueOf(fields[3]) * 100;
                             final int expectedId;
-                            if (fields[1].equals("1")) {
+                            if (major > 0 && major < 6000000) {
                                 expectedId = 0x08000000 ^ (major + minor + revision + 99);
                             } else {
                                 expectedId = (major + minor + revision + 99);

server/src/main/java/org/opensearch/Version.java

@@ -238,8 +238,8 @@ public boolean onOrBefore(Version version) {
 
     // LegacyESVersion major 7 is equivalent to Version major 1
     public int compareMajor(Version other) {
-        int m = major == 1 ? 7 : major;
-        int om = other.major == 1 ? 7 : other.major;
+        int m = major == 1 ? 7 : major == 2 ? 8 : major;
+        int om = other.major == 1 ? 7 : other.major == 2 ? 8 : other.major;
         return Integer.compare(m, om);
     }
 
@@ -293,6 +293,8 @@ protected Version computeMinCompatVersion() {
         } else if (major == 6) {
             // force the minimum compatibility for version 6 to 5.6 since we don't reference version 5 anymore
             return Version.fromId(5060099);
+        } else if (major == 2) {
+            return LegacyESVersion.V_7_10_0;
         } else if (major >= 7) {
             // all major versions from 7 onwards are compatible with last minor series of the previous major
             Version bwcVersion = null;
@@ -339,6 +341,8 @@ private Version computeMinIndexCompatVersion() {
             bwcMajor = 2; // we jumped from 2 to 5
         } else if (major == 7 || major == 1) {
             return LegacyESVersion.V_6_0_0_beta1;
+        } else if (major == 2) {
+            return LegacyESVersion.V_7_0_0;
         } else {
             bwcMajor = major - 1;
         }
@@ -354,8 +358,20 @@ public boolean isCompatible(Version version) {
             && version.onOrAfter(minimumCompatibilityVersion());
 
         // OpenSearch version 1 is the functional equivalent of predecessor version 7
-        int a = major == 1 ? 7 : major;
-        int b = version.major == 1 ? 7 : version.major;
+        // OpenSearch version 2 is the functional equivalent of predecessor unreleased version "8"
+        // todo refactor this logic after removing deprecated features
+        int a = major;
+        if (major == 1) {
+            a = 7;
+        } else if (major == 2) {
+            a = 8;
+        }
+        int b = version.major;
+        if (version.major == 1) {
+            b = 7;
+        } else if (version.major == 2) {
+            b = 8;
+        }
 
         assert compatible == false || Math.max(a, b) - Math.min(a, b) <= 1;
         return compatible;

server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java

@@ -367,9 +367,8 @@ public static void ensureNodesCompatibility(Version joiningNodeVersion, Version
      * version mode
      **/
     public static void ensureMajorVersionBarrier(Version joiningNodeVersion, Version minClusterNodeVersion) {
-        final byte jnMajor = joiningNodeVersion.major == 1 ? 7 : joiningNodeVersion.major;
         final byte clusterMajor = minClusterNodeVersion.major == 1? 7: minClusterNodeVersion.major;
-        if (jnMajor < clusterMajor) {
+        if (joiningNodeVersion.compareMajor(minClusterNodeVersion) < 0) {
             throw new IllegalStateException("node version [" + joiningNodeVersion + "] is not supported. " +
                 "All nodes in the cluster are of a higher major [" + clusterMajor + "].");
         }

server/src/test/java/org/opensearch/VersionTests.java

@@ -211,6 +211,48 @@ public void testMinCompatVersion() {
         assertEquals(0, LegacyESVersion.V_7_0_0.minimumCompatibilityVersion().revision);
     }
 
+    /** test opensearch min wire compatibility */
+    public void testOpenSearchMinCompatVersion() {
+        Version opensearchVersion = VersionUtils.randomOpenSearchVersion(random());
+        // opensearch 1.x minCompat is Legacy 6.8.0
+        // opensearch 2.x minCompat is Legacy 7.10.0
+        // opensearch 3.x minCompat is 1.{last minor version}.0
+        // until 3.0 is staged the following line will only return legacy versions
+        List<Version> candidates = opensearchVersion.major >= 3 ? VersionUtils.allOpenSearchVersions() : VersionUtils.allLegacyVersions();
+        int opensearchMajor = opensearchVersion.major;
+        int major = opensearchMajor - 1;
+        if (opensearchMajor == 1) {
+            major = 7;
+        } else if (opensearchMajor == 2) {
+            major = 8;
+        }
+        assertEquals(VersionUtils.lastFirstReleasedMinorFromMajor(candidates, major - 1),
+            opensearchVersion.minimumCompatibilityVersion());
+    }
+
+    /** test opensearch min index compatibility */
+    public void testOpenSearchMinIndexCompatVersion() {
+        Version opensearchVersion = VersionUtils.randomOpenSearchVersion(random());
+        // opensearch 1.x minIndexCompat is Legacy 6.8.0
+        // opensearch 2.x minCompat is Legacy 7.10.0
+        // opensearch 3.x minCompat is 1.{last minor version}.0
+        // until 3.0 is staged the following line will only return legacy versions
+        List<Version> candidates = opensearchVersion.major >= 3 ? VersionUtils.allOpenSearchVersions() : VersionUtils.allLegacyVersions();
+        int opensearchMajor = opensearchVersion.major;
+        int major = opensearchMajor - 1;
+        if (opensearchMajor == 1) {
+            major = 7;
+        } else if (opensearchMajor == 2) {
+            major = 8;
+        }
+        Version expected = VersionUtils.getFirstVersionOfMajor(candidates, major - 1);
+        Version actual = opensearchVersion.minimumIndexCompatibilityVersion();
+        // since some legacy versions still support build (alpha, beta, RC) we check major minor revision only
+        assertEquals(expected.major, actual.major);
+        assertEquals(expected.minor, actual.minor);
+        assertEquals(expected.revision, actual.revision);
+    }
+
     public void testToString() {
         // with 2.0.beta we lowercase
         assertEquals("2.0.0-beta1", LegacyESVersion.fromString("2.0.0-beta1").toString());

server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java

@@ -88,8 +88,7 @@ public void testPreventJoinClusterWithUnsupportedIndices() {
         Settings.builder().build();
         Metadata.Builder metaBuilder = Metadata.builder();
         IndexMetadata indexMetadata = IndexMetadata.builder("test")
-            .settings(settings(VersionUtils.getPreviousVersion(Version.CURRENT
-                .minimumIndexCompatibilityVersion())))
+            .settings(settings(Version.fromString("5.8.0")))
             .numberOfShards(1)
             .numberOfReplicas(1).build();
         metaBuilder.put(indexMetadata, false);

test/framework/src/main/java/org/opensearch/test/VersionUtils.java

@@ -39,6 +39,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -149,6 +150,8 @@ private static Version moveLastToUnreleased(List<List<Version>> versions, List<V
     private static final List<Version> RELEASED_VERSIONS;
     private static final List<Version> UNRELEASED_VERSIONS;
     private static final List<Version> ALL_VERSIONS;
+    private static final List<Version> ALL_OPENSEARCH_VERSIONS;
+    private static final List<Version> ALL_LEGACY_VERSIONS;
 
     static {
         Tuple<List<Version>, List<Version>> versions = resolveReleasedVersions(Version.CURRENT, LegacyESVersion.class);
@@ -159,6 +162,9 @@ private static Version moveLastToUnreleased(List<List<Version>> versions, List<V
         allVersions.addAll(UNRELEASED_VERSIONS);
         Collections.sort(allVersions);
         ALL_VERSIONS = Collections.unmodifiableList(allVersions);
+        // @todo remove this when legacy support is no longer needed
+        ALL_OPENSEARCH_VERSIONS = ALL_VERSIONS.stream().filter(v -> v.major < 6).collect(Collectors.toList());
+        ALL_LEGACY_VERSIONS = ALL_VERSIONS.stream().filter(v -> v.major >= 6).collect(Collectors.toList());
     }
 
     /**
@@ -182,6 +188,16 @@ public static List<Version> allVersions() {
         return ALL_VERSIONS;
     }
 
+    /** Returns an immutable, sorted list containing all opensearch versions; released and unreleased */
+    public static List<Version> allOpenSearchVersions() {
+        return ALL_OPENSEARCH_VERSIONS;
+    }
+
+    /** Returns an immutable, sorted list containing all legacy versions; released and unreleased */
+    public static List<Version> allLegacyVersions() {
+        return ALL_LEGACY_VERSIONS;
+    }
+
     /**
      * Get the released version before {@code version}.
      */
@@ -224,11 +240,35 @@ public static Version getFirstVersion() {
         return RELEASED_VERSIONS.get(0);
     }
 
+    public static Version getFirstVersionOfMajor(List<Version> versions, int major) {
+        Map<Integer, List<Version>> majorVersions = versions.stream().collect(Collectors.groupingBy(v -> (int)v.major));
+        return majorVersions.get(major).get(0);
+    }
+
     /** Returns a random {@link Version} from all available versions. */
     public static Version randomVersion(Random random) {
         return ALL_VERSIONS.get(random.nextInt(ALL_VERSIONS.size()));
     }
 
+    /**
+     * Return a random {@link Version} from all available opensearch versions.
+     **/
+    public static Version randomOpenSearchVersion(Random random) {
+        return ALL_OPENSEARCH_VERSIONS.get(random.nextInt(ALL_OPENSEARCH_VERSIONS.size()));
+    }
+
+    /** Returns the first released (e.g., patch version 0) {@link Version} of the last minor from the requested major version
+     *  e.g., for version 1.0.0 this would be legacy version (7.10.0); the first release (patch 0), of the last
+     *  minor (for 7.x that is minor version 10) for the desired major version (7)
+     **/
+    public static Version lastFirstReleasedMinorFromMajor(List<Version> allVersions, int major) {
+        Map<Integer, List<Version>> majorVersions = allVersions.stream().collect(Collectors.groupingBy(v -> (int)v.major));
+        Map<Integer, List<Version>> groupedByMinor = majorVersions.get(major).stream().collect(
+            Collectors.groupingBy(v -> (int)v.minor));
+        List<Version> candidates = Collections.max(groupedByMinor.entrySet(), Comparator.comparing(Map.Entry::getKey)).getValue();
+        return candidates.get(0);
+    }
+
     /** Returns a random {@link Version} from all available versions, that is compatible with the given version. */
     public static Version randomCompatibleVersion(Random random, Version version) {
         final List<Version> compatible = ALL_VERSIONS.stream().filter(version::isCompatible).collect(Collectors.toList());