Merge branch 'main' into dependabot/gradle/plugins/repository-gcs/io.…

…opencensus-opencensus-api-0.31.1

Signed-off-by: Tianli Feng <ftianli@amazon.com>

.github/dependabot.yml

@@ -207,6 +207,14 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
+  - directory: /distribution/archives/darwin-arm64-tar/
+    open-pull-requests-limit: 1
+    package-ecosystem: gradle
+    schedule:
+      interval: weekly
+    labels:
+      - "dependabot"
+      - "dependencies"
   - directory: /distribution/archives/integ-test-zip/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/pull_request_template.md

@@ -8,8 +8,9 @@
 ### Description
 [Describe what this change achieves]
 
-### Issues Resolved
-[List any issues this PR will resolve]
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
 
 ### Check List
 - [ ] New functionality includes testing.

.github/workflows/version.yml

@@ -105,7 +105,7 @@ jobs:
           echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
           sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
           echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
-          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" server/src/main/java/org/opensearch/Version.java
+          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -40,10 +41,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.11 to 0.16.12
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `org.apache.commons:commons-configuration2` from 2.8.0 to 2.9.0
-- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.2.0
-- Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.2.0
+- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
+- Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.3.0
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `com.google.protobuf:protobuf-java` from 3.22.2 to 3.22.3
+- Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
 
 ### Changed
@@ -76,16 +78,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
+- Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
 - [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
+- Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244))
 
 ### Dependencies
+- Bump `jackson` from 2.14.2 to 2.15.0 ([#7286](https://github.com/opensearch-project/OpenSearch/pull/7286)
 
 ### Changed
+- Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
 
 ### Deprecated
 
@@ -96,4 +102,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x

build.gradle

@@ -53,7 +53,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.17.0" apply false
+  id "com.diffplug.spotless" version "6.18.0" apply false
   id "org.gradle.test-retry" version "1.5.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
@@ -225,8 +225,16 @@ tasks.register("verifyVersions") {
  */
 
 boolean bwc_tests_enabled = true
+
 /* place an issue link here when committing bwc changes */
-final String bwc_tests_disabled_issue = ""
+String bwc_tests_disabled_issue = ""
+
+/* there's no existing MacOS release, therefore disable bcw tests */
+if (Os.isFamily(Os.FAMILY_MAC)) {
+  bwc_tests_enabled = false
+  bwc_tests_disabled_issue = "https://github.com/opensearch-project/OpenSearch/issues/4173"
+}
+
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -106,7 +106,7 @@ dependencies {
   api 'org.apache.commons:commons-compress:1.22'
   api 'org.apache.ant:ant:1.10.13'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0'
-  api 'com.netflix.nebula:nebula-publishing-plugin:20.2.0'
+  api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
   api 'com.netflix.nebula:gradle-info-plugin:12.1.0'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.11.0'
@@ -187,6 +187,7 @@ if (project != rootProject) {
   dependencies {
     reaper project('reaper')
     distribution project(':distribution:archives:darwin-tar')
+    distribution project(':distribution:archives:darwin-arm64-tar')
     distribution project(':distribution:archives:linux-arm64-tar')
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:windows-zip')

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -74,8 +74,8 @@
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
-    private static final String DEFAULT_LEGACY_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/opensearch/LegacyESVersion.java";
-    private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/opensearch/Version.java";
+    private static final String DEFAULT_LEGACY_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/LegacyESVersion.java";
+    private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "libs/core/src/main/java/org/opensearch/Version.java";
     private static Integer _defaultParallel = null;
 
     private final JvmMetadataDetector jvmMetadataDetector;
@@ -152,7 +152,7 @@ private static BwcVersions resolveBwcVersions(File root) {
             versionLines.addAll(IOUtils.readLines(fis2, "UTF-8"));
             return new BwcVersions(versionLines);
         } catch (IOException e) {
-            throw new IllegalStateException("Unable to resolve to resolve bwc versions from versionsFile.", e);
+            throw new IllegalStateException("Unable to resolve bwc versions from versionsFile.", e);
         }
     }
 

buildSrc/version.properties

@@ -8,8 +8,8 @@ bundled_jdk = 19.0.2+7
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.14.2
-jackson_databind  = 2.14.2
+jackson           = 2.15.0
+jackson_databind  = 2.15.0
 snakeyaml         = 2.0
 icu4j             = 70.1
 supercsv          = 2.4.0

client/rest-high-level/src/main/java/org/opensearch/client/indices/IndexTemplateMetadata.java

@@ -36,14 +36,15 @@
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.Nullable;
 import org.opensearch.core.ParseField;
-import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.ConstructingObjectParser;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.mapper.MapperService;
 
 import java.io.IOException;
 import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -59,9 +60,7 @@ public class IndexTemplateMetadata {
         true,
         (a, name) -> {
             List<Map.Entry<String, AliasMetadata>> alias = (List<Map.Entry<String, AliasMetadata>>) a[5];
-            ImmutableOpenMap<String, AliasMetadata> aliasMap = new ImmutableOpenMap.Builder<String, AliasMetadata>().putAll(
-                alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
-            ).build();
+            final Map<String, AliasMetadata> aliasMap = alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
             return new IndexTemplateMetadata(
                 name,
                 (Integer) a[0],
@@ -128,7 +127,7 @@ public class IndexTemplateMetadata {
 
     private final MappingMetadata mappings;
 
-    private final ImmutableOpenMap<String, AliasMetadata> aliases;
+    private final Map<String, AliasMetadata> aliases;
 
     public IndexTemplateMetadata(
         String name,
@@ -137,7 +136,7 @@ public IndexTemplateMetadata(
         List<String> patterns,
         Settings settings,
         MappingMetadata mappings,
-        ImmutableOpenMap<String, AliasMetadata> aliases
+        final Map<String, AliasMetadata> aliases
     ) {
         if (patterns == null || patterns.isEmpty()) {
             throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns);
@@ -148,7 +147,7 @@ public IndexTemplateMetadata(
         this.patterns = patterns;
         this.settings = settings;
         this.mappings = mappings;
-        this.aliases = aliases;
+        this.aliases = Collections.unmodifiableMap(aliases);
     }
 
     public String name() {
@@ -176,7 +175,7 @@ public MappingMetadata mappings() {
         return this.mappings;
     }
 
-    public ImmutableOpenMap<String, AliasMetadata> aliases() {
+    public Map<String, AliasMetadata> aliases() {
         return this.aliases;
     }
 
@@ -217,12 +216,12 @@ public static class Builder {
 
         private MappingMetadata mappings;
 
-        private final ImmutableOpenMap.Builder<String, AliasMetadata> aliases;
+        private final Map<String, AliasMetadata> aliases;
 
         public Builder(String name) {
             this.name = name;
             mappings = null;
-            aliases = ImmutableOpenMap.builder();
+            aliases = new HashMap<>();
         }
 
         public Builder(IndexTemplateMetadata indexTemplateMetadata) {
@@ -233,7 +232,7 @@ public Builder(IndexTemplateMetadata indexTemplateMetadata) {
             settings(indexTemplateMetadata.settings());
 
             mappings = indexTemplateMetadata.mappings();
-            aliases = ImmutableOpenMap.builder(indexTemplateMetadata.aliases());
+            aliases = new HashMap<>(indexTemplateMetadata.aliases());
         }
 
         public Builder order(int order) {
@@ -277,7 +276,7 @@ public Builder putAlias(AliasMetadata.Builder aliasMetadata) {
         }
 
         public IndexTemplateMetadata build() {
-            return new IndexTemplateMetadata(name, order, version, indexPatterns, settings, mappings, aliases.build());
+            return new IndexTemplateMetadata(name, order, version, indexPatterns, settings, mappings, aliases);
         }
 
         public static IndexTemplateMetadata fromXContent(XContentParser parser, String templateName) throws IOException {

client/rest-high-level/src/test/java/org/opensearch/client/AbstractResponseTestCase.java

@@ -32,7 +32,6 @@
 package org.opensearch.client;
 
 import org.opensearch.common.bytes.BytesReference;
-import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.ToXContent;
@@ -43,10 +42,6 @@
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
 
 /**
  * Base class for HLRC response parsing tests.
@@ -103,17 +98,4 @@ public final void testFromXContent() throws IOException {
     protected ToXContent.Params getParams() {
         return ToXContent.EMPTY_PARAMS;
     }
-
-    protected static <T> void assertMapEquals(ImmutableOpenMap<String, T> expected, Map<String, T> actual) {
-        Set<String> expectedKeys = new HashSet<>();
-        Iterator<String> keysIt = expected.keysIt();
-        while (keysIt.hasNext()) {
-            expectedKeys.add(keysIt.next());
-        }
-
-        assertEquals(expectedKeys, actual.keySet());
-        for (String key : expectedKeys) {
-            assertEquals(expected.get(key), actual.get(key));
-        }
-    }
 }

client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexResponseTests.java

@@ -37,7 +37,6 @@
 import org.opensearch.client.GetAliasesResponseTests;
 import org.opensearch.cluster.metadata.AliasMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.XContentParser;
@@ -63,10 +62,10 @@ public class GetIndexResponseTests extends AbstractResponseTestCase<
     protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerTestInstance(XContentType xContentType) {
         String[] indices = generateRandomStringArray(5, 5, false, false);
         final Map<String, MappingMetadata> mappings = new HashMap<>();
-        ImmutableOpenMap.Builder<String, List<AliasMetadata>> aliases = ImmutableOpenMap.builder();
-        ImmutableOpenMap.Builder<String, Settings> settings = ImmutableOpenMap.builder();
-        ImmutableOpenMap.Builder<String, Settings> defaultSettings = ImmutableOpenMap.builder();
-        ImmutableOpenMap.Builder<String, String> dataStreams = ImmutableOpenMap.builder();
+        final Map<String, List<AliasMetadata>> aliases = new HashMap<>();
+        final Map<String, Settings> settings = new HashMap<>();
+        final Map<String, Settings> defaultSettings = new HashMap<>();
+        final Map<String, String> dataStreams = new HashMap<>();
         IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS;
         boolean includeDefaults = randomBoolean();
         for (String index : indices) {
@@ -95,10 +94,10 @@ protected org.opensearch.action.admin.indices.get.GetIndexResponse createServerT
         return new org.opensearch.action.admin.indices.get.GetIndexResponse(
             indices,
             mappings,
-            aliases.build(),
-            settings.build(),
-            defaultSettings.build(),
-            dataStreams.build()
+            aliases,
+            settings,
+            defaultSettings,
+            dataStreams
         );
     }
 
@@ -114,9 +113,9 @@ protected void assertInstances(
     ) {
         assertArrayEquals(serverTestInstance.getIndices(), clientInstance.getIndices());
         assertEquals(serverTestInstance.getMappings(), clientInstance.getMappings());
-        assertMapEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
-        assertMapEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
-        assertMapEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
+        assertEquals(serverTestInstance.getSettings(), clientInstance.getSettings());
+        assertEquals(serverTestInstance.defaultSettings(), clientInstance.getDefaultSettings());
+        assertEquals(serverTestInstance.getAliases(), clientInstance.getAliases());
     }
 
     private static MappingMetadata createMappingsForIndex() {

client/rest-high-level/src/test/java/org/opensearch/client/indices/GetIndexTemplatesResponseTests.java

@@ -137,10 +137,10 @@ public void testParsingFromOpenSearchResponse() throws IOException {
                     assertThat(result.mappings().sourceAsMap(), equalTo(expectedMapping.get("_doc")));
 
                     assertThat(result.aliases().size(), equalTo(esIMD.aliases().size()));
-                    List<AliasMetadata> expectedAliases = Arrays.stream(esIMD.aliases().values().toArray(AliasMetadata.class))
+                    List<AliasMetadata> expectedAliases = Arrays.stream(esIMD.aliases().values().toArray(new AliasMetadata[0]))
                         .sorted(Comparator.comparing(AliasMetadata::alias))
                         .collect(Collectors.toList());
-                    List<AliasMetadata> actualAliases = Arrays.stream(result.aliases().values().toArray(AliasMetadata.class))
+                    List<AliasMetadata> actualAliases = Arrays.stream(result.aliases().values().toArray(new AliasMetadata[0]))
                         .sorted(Comparator.comparing(AliasMetadata::alias))
                         .collect(Collectors.toList());
                     for (int j = 0; j < result.aliases().size(); j++) {
@@ -216,7 +216,7 @@ static void toXContent(GetIndexTemplatesResponse response, XContentBuilder build
 
             serverTemplateBuilder.patterns(clientITMD.patterns());
 
-            Iterator<AliasMetadata> aliases = clientITMD.aliases().valuesIt();
+            Iterator<AliasMetadata> aliases = clientITMD.aliases().values().iterator();
             aliases.forEachRemaining((a) -> serverTemplateBuilder.putAlias(a));
 
             serverTemplateBuilder.settings(clientITMD.settings());

client/sniffer/licenses/jackson-core-2.15.0.jar.sha1

@@ -0,0 +1 @@
+12f334a1dc9c6d2854c43ae314024dde8b3ad572

distribution/build.gradle

@@ -344,7 +344,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     modulesFiles = { platform ->
       copySpec {
         eachFile {
-          if (it.relativePath.segments[-2] == 'bin' || (platform == 'darwin-x64' && it.relativePath.segments[-2] == 'MacOS')) {
+          if (it.relativePath.segments[-2] == 'bin' || ((platform == 'darwin-x64' || platform == 'darwin-arm64') && it.relativePath.segments[-2] == 'MacOS')) {
             // bin files, wherever they are within modules (eg platform specific) should be executable
             // and MacOS is an alternative to bin on macOS
             it.mode = 0755
@@ -622,6 +622,7 @@ subprojects {
 }
 
 ['archives:darwin-tar',
+ 'archives:darwin-arm64-tar',
  'archives:integ-test-zip',
  'archives:linux-arm64-tar',
  'archives:linux-tar',

distribution/docker/build.gradle

@@ -33,6 +33,7 @@ configurations {
 }
 
 dependencies {
+  arm64DockerSource project(path: ":distribution:archives:darwin-arm64-tar", configuration:"default")
   arm64DockerSource project(path: ":distribution:archives:linux-arm64-tar", configuration:"default")
   s390xDockerSource project(path: ":distribution:archives:linux-s390x-tar", configuration:"default")
   ppc64leDockerSource project(path: ":distribution:archives:linux-ppc64le-tar", configuration:"default")