Upgrade Lucene to a 9.7.0 snapshot (#96433)

Most relevant changes: - add api to allow concurrent query rewrite (GITHUB-11838 Add api to allow concurrent query rewrite apache/lucene#11840) - knn query rewrite (Concurrent rewrite for KnnVectorQuery apache/lucene#12160) - Integrate the incubating Panama Vector API (Integrate the Incubating Panama Vector API apache/lucene#12311) As part of this commit I moved the ES codebase off of overriding or relying on the deprecated rewrite(IndexReader) method in favour of using rewrite(IndexSearcher) instead. For score functions, I went for not breaking existing plugins and create a new IndexSearcher whenever we rewrite a filter, otherwise we'd need to change the ScoreFunction#rewrite signature to take a searcher instead of a reader. Co-authored-by: ChrisHegarty <christopher.hegarty@elastic.co>
elastic · May 31, 2023 · e5768d9 · e5768d9
1 parent 47c70f9
commit e5768d9
Show file tree

Hide file tree

Showing 33 changed files with 191 additions and 161 deletions.
diff --git a/...est/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy b/...est/groovy/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTaskFuncTest.groovy
@@ -97,7 +97,6 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleInternalPluginFuncTest {
 
         def output = normalized(result.getOutput())
         assertOutputContains(output, """\
-            Forbidden APIs output:
             DEBUG: Classpath: [file:./build/precommit/thirdPartyAudit/thirdPartyAudit/]
             DEBUG: Detected Java 9 or later with module system.
             ERROR: Forbidden class/interface use: java.io.File [non-public internal runtime class]
@@ -135,7 +134,6 @@ class ThirdPartyAuditTaskFuncTest extends AbstractGradleInternalPluginFuncTest {
 
         def output = normalized(result.getOutput())
         assertOutputContains(output, """\
-            Forbidden APIs output:
             DEBUG: Classpath: [file:./build/precommit/thirdPartyAudit/thirdPartyAudit/]
             DEBUG: Detected Java 9 or later with module system.
             DEBUG: Class 'org.apache.logging.log4j.LogManager' cannot be loaded (while looking up details about referenced class 'org.apache.logging.log4j.LogManager').

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -11,6 +11,8 @@
 
 import org.apache.commons.io.output.NullOutputStream;
 import org.elasticsearch.gradle.OS;
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.internal.info.BuildParams;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.file.ArchiveOperations;
@@ -55,6 +57,8 @@
 
 import javax.inject.Inject;
 
+import static org.gradle.api.JavaVersion.VERSION_20;
+
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
 
@@ -333,6 +337,10 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), classpath);
+            // Enable explicitly for each release as appropriate. Just JDK 20 for now, and just the vector module.
+            if (isJava20()) {
+                spec.jvmArgs("--add-modules", "jdk.incubator.vector");
+            }
             spec.jvmArgs("-Xmx1g");
             spec.getMainClass().set("de.thetaphi.forbiddenapis.cli.CliMain");
             spec.args("-f", getSignatureFile().getAbsolutePath(), "-d", getJarExpandDir(), "--debug", "--allowmissingclasses");
@@ -355,6 +363,18 @@ private String runForbiddenAPIsCli() throws IOException {
         return forbiddenApisOutput;
     }
 
+    /** Returns true iff the Java version is 20. */
+    private boolean isJava20() {
+        if (BuildParams.getIsRuntimeJavaHomeSet()) {
+            if (VERSION_20.equals(BuildParams.getRuntimeJavaVersion())) {
+                return true;
+            }
+        } else if ("20".equals(VersionProperties.getBundledJdkMajorVersion())) {
+            return true;
+        }
+        return false;
+    }
+
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = execOperations.javaexec(spec -> {

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.9.0
-lucene            = 9.6.0
+lucene            = 9.7.0-snapshot-24df30cca69
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 20.0.1+9@b4887098932d415489976708ad6d1a4b

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.6.0
-:lucene_version_path:   9_6_0
+:lucene_version:        9.7.0
+:lucene_version_path:   9_7_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

diff --git a/docs/changelog/96433.yaml b/docs/changelog/96433.yaml
@@ -0,0 +1,5 @@
+pr: 96433
+summary: Upgrade Lucene to a 9.7.0 snapshot
+area: Search
+type: upgrade
+issues: []
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -2483,124 +2483,124 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.6.0">
-         <artifact name="lucene-analysis-common-9.6.0.jar">
-            <sha256 value="f2f0d724c99440f4ab9304c06e7966653d088e872a2bbd709605dff547e5a494" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-common-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="5c5a049f51a233d2a3f0c50bdb625b52905c1c537b3e38e040e6517bfa074b2b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.6.0">
-         <artifact name="lucene-analysis-icu-9.6.0.jar">
-            <sha256 value="7bb19522d55135bfb5a83ecd5fd0df0f9d06166f1a6cf1ca8fdb3173c4c3aa54" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-icu-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="6e378b9a818699e6fcb3cdda4187a4672bfa47b2de288a3c8f258b243954f476" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.6.0">
-         <artifact name="lucene-analysis-kuromoji-9.6.0.jar">
-            <sha256 value="1fc0e3cc75b4ee91a9b8c822e91f4af651ad01da4e90da4acd3c38fb63a9b191" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-kuromoji-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="b05149a2feae54626e870a7829e8b00245f59be5985809e1c55830e5b2f219cc" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.6.0">
-         <artifact name="lucene-analysis-morfologik-9.6.0.jar">
-            <sha256 value="d5d320b1c3923cedfe1239a3f6d32c0591c35c5bed8425d51d2af598628f61b6" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-morfologik-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="aa340253882a997370ddb3a0350f800d48386e496e9362834dbe8dd2eb86e385" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.6.0">
-         <artifact name="lucene-analysis-nori-9.6.0.jar">
-            <sha256 value="05a37f007f423c8ca4977990996bff53162912706b2189499576a65a60cd7ef8" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-nori-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="88259a45357e7ae085cddd3da37352839d0bcb8b3b7f99acc7065859d851ff06" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.6.0">
-         <artifact name="lucene-analysis-phonetic-9.6.0.jar">
-            <sha256 value="2a8150b034f1ce8bf4628ca833d6852f78778bf9b30ae80cd24e114626dc87e5" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-phonetic-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="2dae3e41c203e8049f792e9328f3d291ffed42a0c6dc422e92d0f4b785759194" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.6.0">
-         <artifact name="lucene-analysis-smartcn-9.6.0.jar">
-            <sha256 value="596f03e2f65859e8a3fcc4314bd1b37e09b2cdd3d5d543293c5a1e01485e1428" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-smartcn-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="67ca1d76a228ef9b3d7ecf4522c250f30567c016fb3e352df54d74bcffcf6e95" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.6.0">
-         <artifact name="lucene-analysis-stempel-9.6.0.jar">
-            <sha256 value="497d15f3395f675bcadb467d5148e0b280ef7fd6bb223ce84eeb06c73ea9fb62" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-analysis-stempel-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="d3bafcd84a883451c24731da5144bf018fa819e909cb05776635a7ba11940826" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.6.0">
-         <artifact name="lucene-backward-codecs-9.6.0.jar">
-            <sha256 value="17f3030fe2042d62ab8b65d6bcaf8291a9d2ca59eaea72c2403eb6d7927a195d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-backward-codecs-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="4ae23e18f3ee52ae0eaceffc70daa62023fdc2cc37896408e68c0b5969235167" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-codecs" version="9.6.0">
-         <artifact name="lucene-codecs-9.6.0.jar">
-            <sha256 value="371100cb3d39fb47a1f00846fe9114ec643bc41525102aec853b6ecfdf08571e" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-codecs" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-codecs-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="256d0c240e96d645d872de342cbcfda3a8ee55e4ee34312d4d2214cf0825f70e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-core" version="9.6.0">
-         <artifact name="lucene-core-9.6.0.jar">
-            <sha256 value="43c3905ecc0c41b77694202b5249e538f3988d1c452bb75decdfb5fdb149dd51" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-core" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-core-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="847a6870085fbe3ffb1b61291d1555c1a3b3e4f9f1fc84bb311527822134b93b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-expressions" version="9.6.0">
-         <artifact name="lucene-expressions-9.6.0.jar">
-            <sha256 value="eeabe3db992197780c95922ba5fa1c2c6e580a5ded4c566d6eda1a42145ac176" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-expressions" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-expressions-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="31c06faa38ddd9f529f81b61c789f926c7f19f198ae6eaa13a84c1cc36f94530" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-grouping" version="9.6.0">
-         <artifact name="lucene-grouping-9.6.0.jar">
-            <sha256 value="7610edf698181f385d2d19a5838536f0a76926c816fcb567a2b0e2795163ee37" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-grouping" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-grouping-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="a5f24c0a842575faadb001ad6651fa59c611dc3cba70903d27243ebfc58c264f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-highlighter" version="9.6.0">
-         <artifact name="lucene-highlighter-9.6.0.jar">
-            <sha256 value="dd9ddcb93ba9e5c19d4de5b9b77530a0d4eceb5e2e8f31ac17e905e314a349ce" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-highlighter-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="96c1ceeae0ec2e3a5764f196695c56833b737716235fa51bd55c1fbc07621890" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-join" version="9.6.0">
-         <artifact name="lucene-join-9.6.0.jar">
-            <sha256 value="84b5496e6e1a76e81f32c871f84b663e82cd4f92e4397af668d0902994ebd700" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-join" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-join-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="792e44cf559ebd0c7173ec6947c0305643767504043c30086c17aeea94e3adc5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-memory" version="9.6.0">
-         <artifact name="lucene-memory-9.6.0.jar">
-            <sha256 value="99fd1e7c06867d2f21e2c86143f774df2009222f76e7379902e87577584e63a7" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-memory" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-memory-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="08b97f8a31b002d3805bcc7aa3bfd007c0c3c3a15637d85ed0c2661a39a1b8ed" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-misc" version="9.6.0">
-         <artifact name="lucene-misc-9.6.0.jar">
-            <sha256 value="8386c940435365fd00682d321cb2785025a308cc2ebadba65f5c208e6d231702" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-misc" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-misc-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="75efe101120e3f762ba7924373d489ca95c67cb976ef34967b599c1145cee3ec" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queries" version="9.6.0">
-         <artifact name="lucene-queries-9.6.0.jar">
-            <sha256 value="7c3d67d2450fabde77e550e11b4ed75a27cf13b3fffe6ec51df4482742107b25" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queries" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-queries-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="3396de8adf2932902c1f41a4c5c0ac60864d837d978e11a08387ecd05ae43533" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queryparser" version="9.6.0">
-         <artifact name="lucene-queryparser-9.6.0.jar">
-            <sha256 value="83f4c46f9e5ac7ec3a9aad5fdb05d336c0c4f520528d28a676b65d57508b87f3" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-queryparser-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="2c66472d5c93c2681f15f46e366dc2359e0054e3745e7ea10b2a2cb69996447a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-sandbox" version="9.6.0">
-         <artifact name="lucene-sandbox-9.6.0.jar">
-            <sha256 value="aa240762a4639bf21c599e82c1c6ff16097d6aad619785772db159ed60169b43" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-sandbox-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="56986817350e5b415130316c794dcc16b73f9dd835af311c3a1d28d1f5e86895" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.6.0">
-         <artifact name="lucene-spatial-extras-9.6.0.jar">
-            <sha256 value="1cbd098111b7ff761fc49dfecdcb096268cb0a55399109e0fe5bb434a6663c9a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-spatial-extras-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="fbb2be4c0e10c12903ceb5cf6cc931c07c9f331d2f247ca975781235004a4693" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.6.0">
-         <artifact name="lucene-spatial3d-9.6.0.jar">
-            <sha256 value="4b723e97e388eb0202c9d728d6ade5c359e54ca196b776104495feff2df944b1" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-spatial3d-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="16eb558e5120bcf9c609c8a8f5083176ed260f8af31366a1bbcb9521c78608fa" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-suggest" version="9.6.0">
-         <artifact name="lucene-suggest-9.6.0.jar">
-            <sha256 value="0ddf5f5b0cf06af1bc5f8ab9c8fc404b6d0006d369eadb15745ad78bb24b38c4" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-suggest" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-suggest-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="74b766301bedd21579b473a6a65b280ae5408b7b8144adb5f29d2520494bd40e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-test-framework" version="9.6.0">
-         <artifact name="lucene-test-framework-9.6.0.jar">
-            <sha256 value="0a70e31b06b12ac4d1380040f5ec40cf15e099762a4a8ef549f489492f9e7e0f" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="9.7.0-snapshot-24df30cca69">
+         <artifact name="lucene-test-framework-9.7.0-snapshot-24df30cca69.jar">
+            <sha256 value="0e93af73fc5bd1c836fbe45c2050161064c17df5d2e1c0b5b014280d7647deaa" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.maven" name="maven-model" version="3.6.2">

diff --git a/...-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java b/...-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java
@@ -10,7 +10,6 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermStates;
@@ -189,8 +188,8 @@ public void visit(QueryVisitor visitor) {
     }
 
     @Override
-    public Query rewrite(IndexReader reader) throws IOException {
-        Query inRewritten = in.rewrite(reader);
+    public Query rewrite(IndexSearcher searcher) throws IOException {
+        Query inRewritten = in.rewrite(searcher);
         if (inRewritten != in) {
             return new SourceConfirmedTextQuery(inRewritten, valueFetcherProvider, indexAnalyzer);
         } else if (in instanceof ConstantScoreQuery) {
@@ -203,7 +202,7 @@ public Query rewrite(IndexReader reader) throws IOException {
         } else if (in instanceof MatchNoDocsQuery) {
             return in; // e.g. empty phrase query
         }
-        return super.rewrite(reader);
+        return super.rewrite(searcher);
     }
 
     @Override