Merge remote-tracking branch 'origin/main' into HEAD

.ci/bwcVersions

@@ -53,3 +53,4 @@ BWC_VERSION:
   - "2.3.0"
   - "2.3.1"
   - "2.4.0"
+  - "2.5.0"

.github/pull_request_template.md

@@ -1,3 +1,10 @@
+<!--  Thanks for sending a pull request, here are some tips:
+
+1. If this is a fix for an undisclosed security vulnerability, please STOP. All security vulnerability reporting and fixes should be done as per our security policy https://github.com/opensearch-project/OpenSearch/security/policy
+2. If this is your first time, please read our contributor guidelines: https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md and developer guide https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md
+3. Ensure you have added or ran the appropriate tests for your PR: https://github.com/opensearch-project/OpenSearch/blob/main/TESTING.md
+-->
+
 ### Description
 [Describe what this change achieves]
 

.github/workflows/backport.yml

@@ -22,7 +22,8 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v1.1.4
+        uses: VachaShah/backport@v2.1.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          files_to_skip: 'CHANGELOG.md'

.github/workflows/gradle-check.yml

@@ -91,3 +91,5 @@ jobs:
               * **RESULT:** ${{ env.result }} :x:
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
+              Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
+              Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?

.gitignore

@@ -11,6 +11,7 @@ out/
 !.idea/inspectionProfiles/Project_Default.xml
 !.idea/runConfigurations/Debug_OpenSearch.xml
 !.idea/vcs.xml
+!.idea/icon.svg
 
 # These files are generated in the main tree by IntelliJ
 benchmarks/src/main/generated/*

DEVELOPER_GUIDE.md

@@ -42,21 +42,22 @@
     - [Gradle Plugins](#gradle-plugins)
       - [Distribution Download Plugin](#distribution-download-plugin)
     - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module)
-  - [Misc](#misc)
-    - [git-secrets](#git-secrets)
-      - [Installation](#installation)
-      - [Configuration](#configuration)
   - [Components](#components)
     - [Build libraries & interfaces](#build-libraries--interfaces)
     - [Clients & Libraries](#clients--libraries)
     - [Plugins](#plugins-1)
-    - [Indexing & search](#indexing--search)
+    - [Indexing & Search](#indexing--search)
     - [Aggregations](#aggregations)
     - [Distributed Framework](#distributed-framework)
-  - [Submitting Changes](#submitting-changes)
-  - [Backports](#backports)
-  - [LineLint](#linelint)
-- [Lucene Snapshots](#lucene-snapshots)
+  - [Misc](#misc)
+    - [Git Secrets](#git-secrets)
+      - [Installation](#installation)
+      - [Configuration](#configuration)
+    - [Submitting Changes](#submitting-changes)
+    - [Backports](#backports)
+    - [LineLint](#linelint)
+    - [Lucene Snapshots](#lucene-snapshots)
+    - [Flaky Tests](#flaky-tests)
 
 # Developer Guide
 
@@ -444,37 +445,12 @@ Refer the installed JAR as any other maven artifact, e.g.
 </dependency>
 ```
 
-## Misc
-
-### git-secrets
-
-Security is our top priority. Avoid checking in credentials.
-
-#### Installation
-Install [awslabs/git-secrets](https://github.com/awslabs/git-secrets) by running the following commands.
-```
-git clone https://github.com/awslabs/git-secrets.git
-cd git-secrets
-make install
-```
-
-#### Configuration
-You can configure git secrets per repository, you need to change the directory to the root of the repository and run the following command.
-```
-git secrets --install
-✓ Installed commit-msg hook to .git/hooks/commit-msg
-✓ Installed pre-commit hook to .git/hooks/pre-commit
-✓ Installed prepare-commit-msg hook to .git/hooks/prepare-commit-msg
-```
-Then, you need to apply patterns for git-secrets, you can install the AWS standard patterns by running the following command.
-```
-git secrets --register-aws
-```
-
 ## Components
+
 As you work in the OpenSearch repo you may notice issues getting labeled with component labels.  It's a housekeeping task to help group together similar pieces of work.  You can pretty much ignore it, but if you're curious, here's what the different labels mean:
 
 ### Build libraries & interfaces
+
 Tasks to make sure the build tasks are useful and packaging and distribution are easy.
 
 Includes:
@@ -488,6 +464,7 @@ Includes:
 
 
 ### Clients & Libraries
+
 APIs and communication mechanisms for external connections to OpenSearch.  This includes the “library” directory in OpenSearch (a set of common functions).
 
 Includes:
@@ -497,6 +474,7 @@ Includes:
 - CLI
 
 ### Plugins
+
 Anything touching the plugin infrastructure within core OpenSearch.
 
 Includes:
@@ -506,7 +484,8 @@ Includes:
 - Plugin interfaces
 
 
-### Indexing & search
+### Indexing & Search
+
 The critical path of indexing and search, including:  Measure index and search, performance, Improving the performance of indexing and search, ensure synchronization OpenSearch APIs with upstream Lucene change (e.g. new field types, changing doc values and codex).
 
 Includes:
@@ -517,6 +496,7 @@ Includes:
 - DocValues
 
 ### Aggregations
+
 Making sure OpenSearch can be used as a compute engine.
 
 Includes:
@@ -525,6 +505,7 @@ Includes:
 - Framework
 
 ### Distributed Framework
+
 Work to make sure that OpenSearch can scale in a distributed manner.
 
 Includes:
@@ -536,15 +517,43 @@ Includes:
 - Shard Strategies
 - Circuit Breakers
 
-## Submitting Changes
+## Misc
+
+### Git Secrets
+
+Security is our top priority. Avoid checking in credentials.
+
+#### Installation
+Install [awslabs/git-secrets](https://github.com/awslabs/git-secrets) by running the following commands.
+```
+git clone https://github.com/awslabs/git-secrets.git
+cd git-secrets
+make install
+```
+
+#### Configuration
+You can configure git secrets per repository, you need to change the directory to the root of the repository and run the following command.
+```
+git secrets --install
+✓ Installed commit-msg hook to .git/hooks/commit-msg
+✓ Installed pre-commit hook to .git/hooks/pre-commit
+✓ Installed prepare-commit-msg hook to .git/hooks/prepare-commit-msg
+```
+Then, you need to apply patterns for git-secrets, you can install the AWS standard patterns by running the following command.
+```
+git secrets --register-aws
+```
+
+### Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).
 
-## Backports
+### Backports
 
 The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
 
-## LineLint
+### LineLint
+
 A linter in [`code-hygiene.yml`](.github/workflows/code-hygiene.yml) that validates simple newline and whitespace rules in all sorts of files. It can:
 - Recursively check a directory tree for files that do not end in a newline
 - Automatically fix these files by adding a newline or trimming extra newlines.
@@ -559,7 +568,20 @@ Pass a list of files or directories to limit your search.
 
     linelint README.md LICENSE
 
-# Lucene Snapshots
+### Lucene Snapshots
+
 The Github workflow in [lucene-snapshots.yml](.github/workflows/lucene-snapshots.yml) is a Github worfklow executable by maintainers to build a top-down snapshot build of lucene.
-These snapshots are available to test compatibility with upcoming changes to Lucene by updating the version at [version.properties](buildsrc/version.properties) with the `version-snapshot-sha` version.
-Example: `lucene = 10.0.0-snapshot-2e941fc`.
+These snapshots are available to test compatibility with upcoming changes to Lucene by updating the version at [version.properties](buildsrc/version.properties) with the `version-snapshot-sha` version. Example: `lucene = 10.0.0-snapshot-2e941fc`.
+
+### Flaky Tests
+
+OpenSearch has a very large test suite with long running, often failing (flaky), integration tests. Such individual tests are labelled as [Flaky Random Test Failure](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22). Your help is wanted fixing these!
+
+If you encounter a build/test failure in CI that is unrelated to the change in your pull request, it may be a known flaky test, or a new test failure.
+
+1. Follow failed CI links, and locate the failing test(s).
+2. Copy-paste the failure into a comment of your PR.
+3. Search through [issues](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aopen+is%3Aissue+label%3A%22flaky-test%22) using the name of the failed test for whether this is a known flaky test. 
+5. If an existing issue is found, paste a link to the known issue in a comment to your PR.
+6. If no existing issue is found, open one.
+7. Retry CI via the GitHub UX or by pushing an update to your PR.

README.md

@@ -2,7 +2,13 @@
 
 [![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/opensearch/)
 [![Documentation](https://img.shields.io/badge/documentation-reference-blue)](https://opensearch.org/docs/latest/opensearch/index/)
-[![codecov](https://codecov.io/gh/opensearch-project/OpenSearch/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/OpenSearch)
+[![Code Coverage](https://codecov.io/gh/opensearch-project/OpenSearch/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/OpenSearch)
+[![Untriaged Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/untriaged?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"untriaged")
+[![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
+[![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
+[![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
+[![2.4 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.4.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.4.0")
+[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
 [![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)

buildSrc/build.gradle

@@ -113,9 +113,10 @@ dependencies {
   api "net.java.dev.jna:jna:5.12.1"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'org.jdom:jdom2:2.0.6.1'
-  api 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.7.10'
+  api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.3'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
+  api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.6.2'
   api 'com.networknt:json-schema-validator:1.0.69'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"

buildSrc/src/integTest/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTarIT.java

@@ -35,6 +35,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.tools.ant.taskdefs.condition.Os;
 import org.opensearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.api.GradleException;
 import org.gradle.testkit.runner.GradleRunner;
@@ -52,6 +53,7 @@
 
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assume.assumeFalse;
 
 public class SymbolicLinkPreservingTarIT extends GradleIntegrationTestCase {
 
@@ -60,6 +62,7 @@ public class SymbolicLinkPreservingTarIT extends GradleIntegrationTestCase {
 
     @Before
     public void before() throws IOException {
+        assumeFalse("Skip tar tests on windows.", Os.isFamily(Os.FAMILY_WINDOWS));
         final Path realFolder = temporaryFolder.getRoot().toPath().resolve("real-folder");
         Files.createDirectory(realFolder);
         Files.createFile(realFolder.resolve("file"));

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -29,13 +29,13 @@
 package org.opensearch.gradle.plugin
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import org.gradle.api.tasks.bundling.AbstractArchiveTask
 import org.opensearch.gradle.BuildPlugin
 import org.opensearch.gradle.NoticeTask
 import org.opensearch.gradle.Version
 import org.opensearch.gradle.VersionProperties
 import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin
 import org.opensearch.gradle.info.BuildParams
-import org.opensearch.gradle.plugin.PluginPropertiesExtension
 import org.opensearch.gradle.test.RestTestBasePlugin
 import org.opensearch.gradle.testclusters.RunTask
 import org.opensearch.gradle.util.Util
@@ -134,6 +134,12 @@ class PluginBuildPlugin implements Plugin<Project> {
         }
         project.configurations.getByName('default')
                 .extendsFrom(project.configurations.getByName('runtimeClasspath'))
+        project.tasks.withType(AbstractArchiveTask.class).configureEach { task ->
+            // ignore file timestamps
+            // be consistent in archive file order
+            task.preserveFileTimestamps = false
+            task.reproducibleFileOrder = true
+        }
         // allow running ES with this plugin in the foreground of a build
         project.tasks.register('run', RunTask) {
             dependsOn(project.tasks.bundlePlugin)

buildSrc/src/main/java/org/opensearch/gradle/PublishPlugin.java

@@ -92,7 +92,7 @@ public String call() throws Exception {
                 return String.format(
                     "%s/distributions/%s-%s.pom",
                     project.getBuildDir(),
-                    getArchivesBaseName(project),
+                    pomTask.getName().toLowerCase().contains("zip") ? project.getName() : getArchivesBaseName(project),
                     project.getVersion()
                 );
             }
@@ -130,7 +130,6 @@ public String call() throws Exception {
             publication.getPom().withXml(PublishPlugin::addScmInfo);
 
             if (!publication.getName().toLowerCase().contains("zip")) {
-
                 // have to defer this until archivesBaseName is set
                 project.afterEvaluate(p -> publication.setArtifactId(getArchivesBaseName(project)));
 
@@ -139,6 +138,8 @@ public String call() throws Exception {
                     publication.artifact(project.getTasks().getByName("sourcesJar"));
                     publication.artifact(project.getTasks().getByName("javadocJar"));
                 }
+            } else {
+                project.afterEvaluate(p -> publication.setArtifactId(project.getName()));
             }
 
             generatePomTask.configure(

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -51,7 +51,6 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -220,14 +219,6 @@ private List<JavaHome> getAvailableJavaVersions(JavaVersion minimumCompilerVersi
         return javaVersions;
     }
 
-    private static boolean isCurrentJavaHome(File javaHome) {
-        try {
-            return Files.isSameFile(javaHome.toPath(), Jvm.current().getJavaHome().toPath());
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     private static String getTestSeed() {
         String testSeedProperty = System.getProperty("tests.seed");
         final String testSeed;

buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java

@@ -73,12 +73,14 @@ public void apply(Project project) {
             .create("distributionArchiveCheck", DistributionArchiveCheckExtension.class);
 
         File archiveExtractionDir = calculateArchiveExtractionDir(project);
-
         // sanity checks if archives can be extracted
         TaskProvider<Copy> checkExtraction = registerCheckExtractionTask(project, buildDistTask, archiveExtractionDir);
+        checkExtraction.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
         TaskProvider<Task> checkLicense = registerCheckLicenseTask(project, checkExtraction);
+        checkLicense.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
 
         TaskProvider<Task> checkNotice = registerCheckNoticeTask(project, checkExtraction);
+        checkNotice.configure(InternalDistributionArchiveSetupPlugin.configure(buildTaskName));
         TaskProvider<Task> checkTask = project.getTasks().named("check");
         checkTask.configure(task -> {
             task.dependsOn(checkExtraction);
@@ -118,7 +120,7 @@ public void execute(Task task) {
     }
 
     private TaskProvider<Task> registerCheckLicenseTask(Project project, TaskProvider<Copy> checkExtraction) {
-        TaskProvider<Task> checkLicense = project.getTasks().register("checkLicense", task -> {
+        return project.getTasks().register("checkLicense", task -> {
             task.dependsOn(checkExtraction);
             task.doLast(new Action<Task>() {
                 @Override
@@ -138,7 +140,6 @@ public void execute(Task task) {
                 }
             });
         });
-        return checkLicense;
     }
 
     private TaskProvider<Copy> registerCheckExtractionTask(Project project, TaskProvider<Task> buildDistTask, File archiveExtractionDir) {