merge upstream main

Signed-off-by: Gao Binlong <gbinlong@amazon.com>
opensearch-project · Mar 23, 2023 · c7eab0f · c7eab0f
2 parents 286faf2 + 4d78bbf
commit c7eab0f
Show file tree

Hide file tree

Showing 402 changed files with 4,436 additions and 3,039 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1 +1 @@
-*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25
diff --git a/.github/workflows/code-hygiene.yml b/.github/workflows/code-hygiene.yml
diff --git a/.github/workflows/github-merit-badger.yml b/.github/workflows/github-merit-badger.yml
@@ -0,0 +1,20 @@
+name: github-merit-badger
+on:
+  pull_request_target:
+    types:
+     - opened
+
+jobs:
+  call-action:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: aws-github-ops/github-merit-badger@v0.0.98
+        id: merit-badger
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          badges: '[first-time-contributor,repeat-contributor,valued-contributor,seasoned-contributor,all-star-contributor,distinguished-contributor]'
+          thresholds: '[0,3,6,13,25,50]'
+          badge-type: 'achievement'
+          ignore-usernames: '[opensearch-ci-bot, dependabot, opensearch-trigger-bot]'
diff --git a/.linelint.yml b/.linelint.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -38,6 +38,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1
 - Bump `com.networknt:json-schema-validator` from 1.0.76 to 1.0.78
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.0.1
+- Bump `org.apache.zookeeper:zookeeper` from 3.8.0 to 3.8.1
+- Bump `net.minidev:json-smart` from 2.4.8 to 2.4.10
+- Bump `org.apache.maven:maven-model` from 3.8.6 to 3.9.1
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -81,18 +84,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Return success on DeletePits when no PITs exist. ([#6544](https://github.com/opensearch-project/OpenSearch/pull/6544))
 - Add node repurpose command for search nodes ([#6517](https://github.com/opensearch-project/OpenSearch/pull/6517))
 - Add wait_for_completion parameter to resize, open, and forcemerge APIs ([#6434](https://github.com/opensearch-project/OpenSearch/pull/6434))
+- [Segment Replication] Apply backpressure when replicas fall behind ([#6563](https://github.com/opensearch-project/OpenSearch/pull/6563))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490))
 - Bump `com.azure:azure-storage-common` from 12.19.3 to 12.20.0 ([#6492](https://github.com/opensearch-project/OpenSearch/pull/6492)
 - Bump `snakeyaml` from 1.33 to 2.0 ([#6511](https://github.com/opensearch-project/OpenSearch/pull/6511))
 - Bump `io.projectreactor.netty:reactor-netty` from 1.1.3 to 1.1.4
+- Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.15.2 to 0.16.11
+- Bump `net.minidev:json-smart` from 2.4.8 to 2.4.9
+- Bump `com.google.protobuf:protobuf-java` from 3.22.0 to 3.22.2
+- Bump Netty to 4.1.90.Final ([#6677](https://github.com/opensearch-project/OpenSearch/pull/6677)
+- Bump `com.diffplug.spotless` from 6.15.0 to 6.17.0
 
 ### Changed
 - Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009))
 - [Refactor] XContent base classes from xcontent to core library ([#5902](https://github.com/opensearch-project/OpenSearch/pull/5902))
 
 ### Deprecated
+- Map, List, and Set in org.opensearch.common.collect ([#6609](https://github.com/opensearch-project/OpenSearch/pull/6609))
 
 ### Removed
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -19,7 +19,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Marc Handalian           | [mch2](https://github.com/mch2)                         | Amazon      |
 | Nick Knize               | [nknize](https://github.com/nknize)                     | Amazon      |
 | Owais Kazi               | [owaiskazi19](https://github.com/owaiskazi19)           | Amazon      |
-| Rabi Panda               | [adnapibar](https://github.com/adnapibar)               | Amazon      |
 | Rishikesh Pasham         | [Rishikesh1159](https://github.com/Rishikesh1159)       | Amazon      |
 | Ryan Bogan               | [ryanbogan](https://github.com/ryanbogan)               | Amazon      |
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
@@ -31,7 +30,8 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 
 ## Emeritus
 
-| Maintainer            | GitHub ID                                 | Affiliation |
-| --------------------- | ----------------------------------------- | ----------- |
-| Abbas Hussain         | [abbashus](https://github.com/abbashus)   | Amazon      |
-| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon      |
+| Maintainer              | GitHub ID                                   | Affiliation |
+|-------------------------|---------------------------------------------| ----------- |
+| Abbas Hussain           | [abbashus](https://github.com/abbashus)     | Amazon      |
+| Megha Sai Kavikondala   | [meghasaik](https://github.com/meghasaik)   | Amazon      |
+| Rabi Panda              | [adnapibar](https://github.com/adnapibar)   | Amazon      |
diff --git a/...nsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java b/...nsearch/benchmark/search/aggregations/bucket/terms/StringTermsSerializationBenchmark.java
@@ -64,9 +64,7 @@
 @State(Scope.Benchmark)
 public class StringTermsSerializationBenchmark {
     private static final NamedWriteableRegistry REGISTRY = new NamedWriteableRegistry(
-        org.opensearch.common.collect.List.of(
-            new NamedWriteableRegistry.Entry(InternalAggregation.class, StringTerms.NAME, StringTerms::new)
-        )
+        List.of(new NamedWriteableRegistry.Entry(InternalAggregation.class, StringTerms.NAME, StringTerms::new))
     );
     @Param(value = { "1000" })
     private int buckets;
@@ -75,15 +73,13 @@ public class StringTermsSerializationBenchmark {
 
     @Setup
     public void initResults() {
-        results = DelayableWriteable.referencing(InternalAggregations.from(org.opensearch.common.collect.List.of(newTerms(true))));
+        results = DelayableWriteable.referencing(InternalAggregations.from(List.of(newTerms(true))));
     }
 
     private StringTerms newTerms(boolean withNested) {
         List<StringTerms.Bucket> resultBuckets = new ArrayList<>(buckets);
         for (int i = 0; i < buckets; i++) {
-            InternalAggregations inner = withNested
-                ? InternalAggregations.from(org.opensearch.common.collect.List.of(newTerms(false)))
-                : InternalAggregations.EMPTY;
+            InternalAggregations inner = withNested ? InternalAggregations.from(List.of(newTerms(false))) : InternalAggregations.EMPTY;
             resultBuckets.add(new StringTerms.Bucket(new BytesRef("test" + i), i, inner, false, 0, DocValueFormat.RAW));
         }
         return new StringTerms(

diff --git a/build.gradle b/build.gradle
@@ -53,7 +53,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.15.0" apply false
+  id "com.diffplug.spotless" version "6.17.0" apply false
   id "org.gradle.test-retry" version "1.5.1" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -115,9 +115,9 @@ dependencies {
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.4'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.11'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
-  api 'org.apache.maven:maven-model:3.8.6'
+  api 'org.apache.maven:maven-model:3.9.1'
   api 'com.networknt:json-schema-validator:1.0.78'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportPlugin.java
@@ -56,13 +56,9 @@ public void apply(Project project) {
 
         Provider<DockerSupportService> dockerSupportServiceProvider = project.getGradle()
             .getSharedServices()
-            .registerIfAbsent(
-                DOCKER_SUPPORT_SERVICE_NAME,
-                DockerSupportService.class,
-                spec -> spec.parameters(
-                    params -> { params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE)); }
-                )
-            );
+            .registerIfAbsent(DOCKER_SUPPORT_SERVICE_NAME, DockerSupportService.class, spec -> spec.parameters(params -> {
+                params.setExclusionsFile(new File(project.getRootDir(), DOCKER_ON_LINUX_EXCLUSIONS_FILE));
+            }));
 
         // Ensure that if we are trying to run any DockerBuildTask tasks, we assert an available Docker installation exists
         project.getGradle().getTaskGraph().whenReady(graph -> {

diff --git a/...dSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/...dSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java
@@ -76,12 +76,9 @@ public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) {
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        BuildParams.getBwcVersions()
-            .forPreviousUnreleased(
-                (BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
-                    configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion);
-                }
-            );
+        BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
+            configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion);
+        });
     }
 
     private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -197,13 +197,9 @@ public void apply(Project project) {
 
             // windows boxes get windows distributions, and linux boxes get linux distributions
             if (isWindows(vmProject)) {
-                configureVMWrapperTasks(
-                    vmProject,
-                    windowsTestTasks,
-                    depsTasks,
-                    wrapperTask -> { vmLifecyleTasks.get(OpenSearchDistribution.Type.ARCHIVE).configure(t -> t.dependsOn(wrapperTask)); },
-                    vmDependencies
-                );
+                configureVMWrapperTasks(vmProject, windowsTestTasks, depsTasks, wrapperTask -> {
+                    vmLifecyleTasks.get(OpenSearchDistribution.Type.ARCHIVE).configure(t -> t.dependsOn(wrapperTask));
+                }, vmDependencies);
             } else {
                 for (Entry<OpenSearchDistribution.Type, List<TaskProvider<Test>>> entry : linuxTestTasks.entrySet()) {
                     OpenSearchDistribution.Type type = entry.getKey();

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -26,7 +26,7 @@ guava             = 31.1-jre
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.87.Final
+netty             = 4.1.90.Final
 joda              = 2.12.2
 
 # client dependencies
@@ -49,9 +49,9 @@ bouncycastle=1.70
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.1.0
+mockito           = 5.2.0
 objenesis         = 3.2
-bytebuddy         = 1.12.22
+bytebuddy         = 1.14.2
 
 # benchmark dependencies
 jmh               = 1.35
diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/core/MainResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/core/MainResponse.java
@@ -43,7 +43,9 @@ public class MainResponse {
     private static final ConstructingObjectParser<MainResponse, Void> PARSER = new ConstructingObjectParser<>(
         MainResponse.class.getName(),
         true,
-        args -> { return new MainResponse((String) args[0], (Version) args[1], (String) args[2], (String) args[3]); }
+        args -> {
+            return new MainResponse((String) args[0], (Version) args[1], (String) args[2], (String) args[3]);
+        }
     );
 
     static {

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/core/TermVectorsResponse.java
@@ -240,7 +240,9 @@ public static final class FieldStatistics {
             private static final ConstructingObjectParser<FieldStatistics, Void> PARSER = new ConstructingObjectParser<>(
                 "field_statistics",
                 true,
-                args -> { return new FieldStatistics((long) args[0], (int) args[1], (long) args[2]); }
+                args -> {
+                    return new FieldStatistics((long) args[0], (int) args[1], (long) args[2]);
+                }
             );
 
             static {
@@ -411,11 +413,9 @@ public int hashCode() {
 
         public static final class Token {
 
-            private static final ConstructingObjectParser<Token, Void> PARSER = new ConstructingObjectParser<>(
-                "token",
-                true,
-                args -> { return new Token((Integer) args[0], (Integer) args[1], (Integer) args[2], (String) args[3]); }
-            );
+            private static final ConstructingObjectParser<Token, Void> PARSER = new ConstructingObjectParser<>("token", true, args -> {
+                return new Token((Integer) args[0], (Integer) args[1], (Integer) args[2], (String) args[3]);
+            });
             static {
                 PARSER.declareInt(optionalConstructorArg(), new ParseField("start_offset"));
                 PARSER.declareInt(optionalConstructorArg(), new ParseField("end_offset"));

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java
@@ -628,10 +628,9 @@ public void testIndex() throws IOException {
             assertEquals("index", indexResponse.getIndex());
             assertEquals("with_create_op_type", indexResponse.getId());
 
-            OpenSearchStatusException exception = expectThrows(
-                OpenSearchStatusException.class,
-                () -> { execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); }
-            );
+            OpenSearchStatusException exception = expectThrows(OpenSearchStatusException.class, () -> {
+                execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync);
+            });
 
             assertEquals(RestStatus.CONFLICT, exception.status());
             assertEquals(

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java
@@ -2031,8 +2031,8 @@ public void testSimulateIndexTemplate() throws Exception {
         Settings settings = Settings.builder().put("index.number_of_shards", 1).build();
         CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"host_name\":{\"type\":\"keyword\"}}}");
         AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build();
-        Template template = new Template(settings, mappings, org.opensearch.common.collect.Map.of("alias", alias));
-        List<String> pattern = org.opensearch.common.collect.List.of("pattern");
+        Template template = new Template(settings, mappings, Map.of("alias", alias));
+        List<String> pattern = List.of("pattern");
         ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(
             pattern,
             template,
@@ -2057,7 +2057,7 @@ public void testSimulateIndexTemplate() throws Exception {
         AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build();
         ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(
             pattern,
-            new Template(null, null, org.opensearch.common.collect.Map.of("simulation-alias", simulationAlias)),
+            new Template(null, null, Map.of("simulation-alias", simulationAlias)),
             Collections.emptyList(),
             2L,
             1L,

diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ReindexIT.java
@@ -112,10 +112,9 @@ public void testReindex() throws IOException {
             reindexRequest.setRefresh(true);
             reindexRequest.setRequireAlias(true);
 
-            OpenSearchStatusException exception = expectThrows(
-                OpenSearchStatusException.class,
-                () -> { execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync); }
-            );
+            OpenSearchStatusException exception = expectThrows(OpenSearchStatusException.class, () -> {
+                execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync);
+            });
             assertEquals(RestStatus.NOT_FOUND, exception.status());
             assertEquals(
                 "OpenSearch exception [type=index_not_found_exception, reason=no such index [dest] and [require_alias] request flag is [true] and [dest] is not an alias]",