Merge branch 'main' into main

benchmarks/src/main/java/org/opensearch/benchmark/time/NanoTimeVsCurrentTimeMillisBenchmark.java

@@ -8,7 +8,15 @@
 
 package org.opensearch.benchmark.time;
 
-import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Benchmark;
 
 import java.util.concurrent.TimeUnit;
 

build.gradle

@@ -320,7 +320,7 @@ allprojects {
     task.reproducibleFileOrder = true
     if (task instanceof SymbolicLinkPreservingTar) {
       // Replace file timestamps with latest Git revision date (if available)
-      task.lastModifiedTimestamp = gitRevisionDate 
+      task.lastModifiedTimestamp = gitRevisionDate
     }
   }
 
@@ -359,7 +359,8 @@ allprojects {
           project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
           String externalLinkName = upstreamProject.archivesBaseName
           String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
-          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
+          String projectRelativePath = project.relativePath(upstreamProject.buildDir)
+          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
         }
       }
       boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)

codecov.yml

@@ -0,0 +1,12 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default:
+        target: 70%    # the required coverage value
+        threshold: 1%  # the leniency in hitting the target

distribution/packages/build.gradle

@@ -392,7 +392,7 @@ tasks.register('buildRpm', Rpm) {
 }
 
 tasks.register('buildNoJdkRpm', Rpm) {
-  configure(commonRpmConfig(true, 'x64'))
+  configure(commonRpmConfig(false, 'x64'))
 }
 
 Closure dpkgExists = { it -> new File('/bin/dpkg-deb').exists() || new File('/usr/bin/dpkg-deb').exists() || new File('/usr/local/bin/dpkg-deb').exists() }

gradle/formatting.gradle

@@ -69,6 +69,13 @@ allprojects {
           eclipse().configFile rootProject.file('buildSrc/formatterConfig.xml')
           trimTrailingWhitespace()
 
+          custom 'Refuse wildcard imports', {
+                // Wildcard imports can't be resolved; fail the build
+                if (it =~ /\s+import .*\*;/) {
+                    throw new AssertionError("Do not use wildcard imports.  'spotlessApply' cannot resolve this issue.")
+                }
+          }
+
           // See DEVELOPER_GUIDE.md for details of when to enable this.
           if (System.getProperty('spotless.paddedcell') != null) {
             paddedCell()

gradle/missing-javadoc.gradle

@@ -70,7 +70,7 @@ allprojects {
       classpath = sourceSets.main.compileClasspath
       srcDirSet = sourceSets.main.java
 
-      outputDir = project.javadoc.destinationDir
+      outputDir = file("${project.buildDir}/tmp/missingJavadoc/")
     }
   }
 }
@@ -183,6 +183,7 @@ configure(project(":server")) {
   }
 }
 
+@CacheableTask
 class MissingJavadocTask extends DefaultTask {
   @InputFiles
   @SkipWhenEmpty
@@ -227,7 +228,8 @@ class MissingJavadocTask extends DefaultTask {
   @Input
   def executable
 
-  @Input
+  @InputFiles
+  @PathSensitive(PathSensitivity.RELATIVE)
   def taskResources
 
   /** Utility method to recursively collect all tasks with same name like this one that we depend on */

modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessLexer.java

@@ -35,10 +35,16 @@
 
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.*;
-import org.antlr.v4.runtime.atn.*;
+import org.antlr.v4.runtime.RuntimeMetaData;
+import org.antlr.v4.runtime.Vocabulary;
+import org.antlr.v4.runtime.VocabularyImpl;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNDeserializer;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
+
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.misc.*;
 
 @SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })
 abstract class PainlessLexer extends Lexer {

modules/lang-painless/src/main/java/org/opensearch/painless/antlr/PainlessParser.java

@@ -32,11 +32,25 @@
  */
 package org.opensearch.painless.antlr;
 
-import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.dfa.DFA;
-import org.antlr.v4.runtime.*;
-import org.antlr.v4.runtime.misc.*;
-import org.antlr.v4.runtime.tree.*;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
+import org.antlr.v4.runtime.atn.ParserATNSimulator;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.ATNDeserializer;
+import org.antlr.v4.runtime.dfa.DFA;
+import org.antlr.v4.runtime.Parser;
+import org.antlr.v4.runtime.RuntimeMetaData;
+import org.antlr.v4.runtime.Vocabulary;
+import org.antlr.v4.runtime.VocabularyImpl;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.TokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.NoViableAltException;
+import org.antlr.v4.runtime.FailedPredicateException;
+import org.antlr.v4.runtime.RuleContext;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.antlr.v4.runtime.tree.ParseTreeVisitor;
 import java.util.List;
 
 @SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast" })

server/src/internalClusterTest/java/org/opensearch/search/profile/query/QueryProfilerIT.java

@@ -34,7 +34,11 @@
 
 import org.apache.lucene.tests.util.English;
 import org.opensearch.action.index.IndexRequestBuilder;
-import org.opensearch.action.search.*;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.MultiSearchResponse;
+import org.opensearch.action.search.SearchType;
+import org.opensearch.action.search.SearchRequestBuilder;
+import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -44,7 +48,11 @@
 import org.opensearch.search.sort.SortOrder;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
-import java.util.*;
+import java.util.List;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.greaterThan;

server/src/internalClusterTest/java/org/opensearch/search/searchafter/SearchAfterIT.java

@@ -32,15 +32,21 @@
 
 package org.opensearch.search.searchafter;
 
+import org.opensearch.action.ActionFuture;
 import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.opensearch.action.index.IndexRequestBuilder;
+import org.opensearch.action.search.CreatePitAction;
+import org.opensearch.action.search.CreatePitRequest;
+import org.opensearch.action.search.CreatePitResponse;
 import org.opensearch.action.search.SearchPhaseExecutionException;
 import org.opensearch.action.search.SearchRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.common.UUIDs;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.search.SearchHit;
+import org.opensearch.search.builder.PointInTimeBuilder;
 import org.opensearch.search.sort.SortOrder;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.hamcrest.Matchers;
@@ -155,6 +161,58 @@ public void testsShouldFail() throws Exception {
         }
     }
 
+    public void testPitWithSearchAfter() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test").setMapping("field1", "type=long", "field2", "type=keyword").get());
+        ensureGreen();
+        indexRandom(
+            true,
+            client().prepareIndex("test").setId("0").setSource("field1", 0),
+            client().prepareIndex("test").setId("1").setSource("field1", 100, "field2", "toto"),
+            client().prepareIndex("test").setId("2").setSource("field1", 101),
+            client().prepareIndex("test").setId("3").setSource("field1", 99)
+        );
+
+        CreatePitRequest request = new CreatePitRequest(TimeValue.timeValueDays(1), true);
+        request.setIndices(new String[] { "test" });
+        ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
+        CreatePitResponse pitResponse = execute.get();
+        SearchResponse sr = client().prepareSearch()
+            .addSort("field1", SortOrder.ASC)
+            .setQuery(matchAllQuery())
+            .searchAfter(new Object[] { 99 })
+            .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+            .get();
+        assertEquals(2, sr.getHits().getHits().length);
+        sr = client().prepareSearch()
+            .addSort("field1", SortOrder.ASC)
+            .setQuery(matchAllQuery())
+            .searchAfter(new Object[] { 100 })
+            .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+            .get();
+        assertEquals(1, sr.getHits().getHits().length);
+        sr = client().prepareSearch()
+            .addSort("field1", SortOrder.ASC)
+            .setQuery(matchAllQuery())
+            .searchAfter(new Object[] { 0 })
+            .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+            .get();
+        assertEquals(3, sr.getHits().getHits().length);
+        /**
+         * Add new data and assert PIT results remain the same and normal search results gets refreshed
+         */
+        indexRandom(true, client().prepareIndex("test").setId("4").setSource("field1", 102));
+        sr = client().prepareSearch()
+            .addSort("field1", SortOrder.ASC)
+            .setQuery(matchAllQuery())
+            .searchAfter(new Object[] { 0 })
+            .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+            .get();
+        assertEquals(3, sr.getHits().getHits().length);
+        sr = client().prepareSearch().addSort("field1", SortOrder.ASC).setQuery(matchAllQuery()).searchAfter(new Object[] { 0 }).get();
+        assertEquals(4, sr.getHits().getHits().length);
+        client().admin().indices().prepareDelete("test").get();
+    }
+
     public void testWithNullStrings() throws InterruptedException {
         assertAcked(client().admin().indices().prepareCreate("test").setMapping("field2", "type=keyword").get());
         ensureGreen();

server/src/internalClusterTest/java/org/opensearch/search/slice/SearchSliceIT.java

@@ -32,9 +32,13 @@
 
 package org.opensearch.search.slice;
 
+import org.opensearch.action.ActionFuture;
 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
 
 import org.opensearch.action.index.IndexRequestBuilder;
+import org.opensearch.action.search.CreatePitAction;
+import org.opensearch.action.search.CreatePitRequest;
+import org.opensearch.action.search.CreatePitResponse;
 import org.opensearch.action.search.SearchPhaseExecutionException;
 import org.opensearch.action.search.SearchRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
@@ -46,6 +50,7 @@
 import org.opensearch.search.Scroll;
 import org.opensearch.search.SearchException;
 import org.opensearch.search.SearchHit;
+import org.opensearch.search.builder.PointInTimeBuilder;
 import org.opensearch.search.sort.SortBuilders;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -86,7 +91,12 @@ private void setupIndex(int numDocs, int numberOfShards) throws IOException, Exe
             client().admin()
                 .indices()
                 .prepareCreate("test")
-                .setSettings(Settings.builder().put("number_of_shards", numberOfShards).put("index.max_slices_per_scroll", 10000))
+                .setSettings(
+                    Settings.builder()
+                        .put("number_of_shards", numberOfShards)
+                        .put("index.max_slices_per_scroll", 10000)
+                        .put("index.max_slices_per_pit", 10000)
+                )
                 .setMapping(mapping)
         );
         ensureGreen();
@@ -129,6 +139,78 @@ public void testSearchSort() throws Exception {
         }
     }
 
+    public void testSearchSortWithoutPitOrScroll() throws Exception {
+        int numShards = randomIntBetween(1, 7);
+        int numDocs = randomIntBetween(100, 1000);
+        setupIndex(numDocs, numShards);
+        int fetchSize = randomIntBetween(10, 100);
+        SearchRequestBuilder request = client().prepareSearch("test")
+            .setQuery(matchAllQuery())
+            .setSize(fetchSize)
+            .addSort(SortBuilders.fieldSort("_doc"));
+        SliceBuilder sliceBuilder = new SliceBuilder("_id", 0, 4);
+        SearchPhaseExecutionException ex = expectThrows(SearchPhaseExecutionException.class, () -> request.slice(sliceBuilder).get());
+        assertTrue(ex.getMessage().contains("all shards failed"));
+    }
+
+    public void testSearchSortWithPIT() throws Exception {
+        int numShards = randomIntBetween(1, 7);
+        int numDocs = randomIntBetween(100, 1000);
+        setupIndex(numDocs, numShards);
+        int max = randomIntBetween(2, numShards * 3);
+        CreatePitRequest pitRequest = new CreatePitRequest(TimeValue.timeValueDays(1), true);
+        pitRequest.setIndices(new String[] { "test" });
+        ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, pitRequest);
+        CreatePitResponse pitResponse = execute.get();
+        for (String field : new String[] { "_id", "random_int", "static_int" }) {
+            int fetchSize = randomIntBetween(10, 100);
+
+            // test _doc sort
+            SearchRequestBuilder request = client().prepareSearch("test")
+                .setQuery(matchAllQuery())
+                .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+                .setSize(fetchSize)
+                .addSort(SortBuilders.fieldSort("_doc"));
+            assertSearchSlicesWithPIT(request, field, max, numDocs);
+
+            // test numeric sort
+            request = client().prepareSearch("test")
+                .setQuery(matchAllQuery())
+                .setPointInTime(new PointInTimeBuilder(pitResponse.getId()))
+                .setSize(fetchSize)
+                .addSort(SortBuilders.fieldSort("random_int"));
+            assertSearchSlicesWithPIT(request, field, max, numDocs);
+        }
+        client().admin().indices().prepareDelete("test").get();
+    }
+
+    private void assertSearchSlicesWithPIT(SearchRequestBuilder request, String field, int numSlice, int numDocs) {
+        int totalResults = 0;
+        List<String> keys = new ArrayList<>();
+        for (int id = 0; id < numSlice; id++) {
+            SliceBuilder sliceBuilder = new SliceBuilder(field, id, numSlice);
+            SearchResponse searchResponse = request.slice(sliceBuilder).setFrom(0).get();
+            totalResults += searchResponse.getHits().getHits().length;
+            int expectedSliceResults = (int) searchResponse.getHits().getTotalHits().value;
+            int numSliceResults = searchResponse.getHits().getHits().length;
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                assertTrue(keys.add(hit.getId()));
+            }
+            while (searchResponse.getHits().getHits().length > 0) {
+                searchResponse = request.setFrom(numSliceResults).slice(sliceBuilder).get();
+                totalResults += searchResponse.getHits().getHits().length;
+                numSliceResults += searchResponse.getHits().getHits().length;
+                for (SearchHit hit : searchResponse.getHits().getHits()) {
+                    assertTrue(keys.add(hit.getId()));
+                }
+            }
+            assertThat(numSliceResults, equalTo(expectedSliceResults));
+        }
+        assertThat(totalResults, equalTo(numDocs));
+        assertThat(keys.size(), equalTo(numDocs));
+        assertThat(new HashSet(keys).size(), equalTo(numDocs));
+    }
+
     public void testWithPreferenceAndRoutings() throws Exception {
         int numShards = 10;
         int totalDocs = randomIntBetween(100, 1000);
@@ -217,7 +299,7 @@ public void testInvalidQuery() throws Exception {
         );
         Throwable rootCause = findRootCause(exc);
         assertThat(rootCause.getClass(), equalTo(SearchException.class));
-        assertThat(rootCause.getMessage(), equalTo("`slice` cannot be used outside of a scroll context"));
+        assertThat(rootCause.getMessage(), equalTo("`slice` cannot be used outside of a scroll context or PIT context"));
     }
 
     private void assertSearchSlicesWithScroll(SearchRequestBuilder request, String field, int numSlice, int numDocs) {