update to maven-indexer 7.0.0.

 - module requires now jdk 11+
 - apache lucene 9.4.1 with memory mapped index
 - can use panama on JDK 19+ if started with --enable-preview
 - multi-threaded remote index extraction (at least twice as fast as ST)
 - bugfix backport hack removed

java/maven.indexer/external/binaries-list

@@ -14,9 +14,9 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-68C3815F7327714D8A75426C30855C960E94AAD5 org.apache.maven.indexer:indexer-core:6.2.2
-75DCF930ECE95581AF3F31AF5692E4963FC7AD8E org.apache.lucene:lucene-core:8.11.1
-558A389A0D68AA7CC4C26B9128C72D0D269D471F org.apache.lucene:lucene-highlighter:8.11.1
-5A44DF2CB26FA1E0E64BE53FE474C7F1D5A3E634 org.apache.lucene:lucene-queryparser:8.11.1
-E50AF506F271A3F7246DA054A2569B42FF73ABB2 org.apache.lucene:lucene-analyzers-common:8.11.1
+D82FEF81AE3B7A99534E90F339937D55C3550881 org.apache.maven.indexer:indexer-core:7.0.0
+48FF655927016066D4746B485D63F0586D859C5F org.apache.lucene:lucene-core:9.4.1
+E97E3003A80C67DDF33A456AB54413B12764D23D org.apache.lucene:lucene-highlighter:9.4.1
+98051F363CD92FD324ACA47B662C4A71B9419E49 org.apache.lucene:lucene-queryparser:9.4.1
+692CBD2E220222ADFF279E922D1C14BB10DAE8B0 org.apache.lucene:lucene-analysis-common:9.4.1
 479C1E06DB31C432330183F5CAE684163F186146 javax.annotation:javax.annotation-api:1.2

java/maven.indexer/external/indexer-core-6.2.2-license.txt → java/maven.indexer/external/indexer-core-7.0.0-license.txt

@@ -1,6 +1,6 @@
 Name: Maven Indexer
 Description: Maven remote repository indexing engine.
-Version: 6.2.2
+Version: 7.0.0
 Origin: Apache Software Foundation
 License: Apache-2.0
 URL: https://repo1.maven.org/maven2/org/apache/maven/indexer/

java/maven.indexer/external/lucene-8.11.1-license.txt → java/maven.indexer/external/lucene-9.4.1-license.txt

@@ -1,11 +1,11 @@
 Name: Apache Lucene
 Description: Java-based indexing and search technology
-Version: 8.11.1
+Version: 9.4.1
 Origin: Apache Software Foundation
 License: Apache-2.0
-URL: http://lucene.apache.org/
-Source: http://svn.apache.org/repos/asf/lucene/java/trunk
-Files: lucene-analyzers-common-8.11.1.jar lucene-core-8.11.1.jar lucene-highlighter-8.11.1.jar lucene-queryparser-8.11.1.jar
+URL: https://lucene.apache.org/
+Source: https://github.com/apache/lucene
+Files: lucene-analysis-common-9.4.1.jar lucene-core-9.4.1.jar lucene-highlighter-9.4.1.jar lucene-queryparser-9.4.1.jar
 
                                  Apache License
                            Version 2.0, January 2004

java/maven.indexer/external/lucene-8.11.1-notice.txt → java/maven.indexer/external/lucene-9.4.1-notice.txt

@@ -1,5 +1,5 @@
 Apache Lucene
-Copyright 2001-2021 The Apache Software Foundation
+Copyright 2001-2022 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
@@ -52,7 +52,7 @@ The snowball stopword lists in
   analysis/common/src/resources/org/apache/lucene/analysis/snowball
 were developed by Martin Porter and Richard Boulton.
 The full snowball package is available from
-  http://snowball.tartarus.org/
+  https://snowballstem.org/
 
 The KStem stemmer in
   analysis/common/src/org/apache/lucene/analysis/en
@@ -91,7 +91,7 @@ and Edmond Nolan.
 The Polish analyzer (stempel) comes with a default
 stopword list that is BSD-licensed created by the Carrot2 project. The file resides
 in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt.
-See http://project.carrot2.org/license.html.
+See https://github.com/carrot2/carrot2.
 
 The SmartChineseAnalyzer source code (smartcn) was
 provided by Xiaoping Gao and copyright 2009 by www.imdict.net.
@@ -101,14 +101,9 @@ is derived from Unicode data such as the Unicode Character Database.
 See http://unicode.org/copyright.html for more details.
 
 The Morfologik analyzer (morfologik) includes BSD-licensed software
-developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/).
-
-Morfologik uses data from Polish ispell/myspell dictionary
-(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia)
-LGPL and Creative Commons ShareAlike.
-
-Morfologic includes data from BSD-licensed dictionary of Polish (SGJP)
-(http://sgjp.pl/morfeusz/)
+developed by Dawid Weiss and Marcin Miłkowski 
+(https://github.com/morfologik/morfologik-stemming) and uses
+data from the BSD-licensed dictionary of Polish (SGJP, http://sgjp.pl/morfeusz/).
 
 Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original
 source code for this can be found at http://www.eclipse.org/jetty/downloads.php
@@ -207,3 +202,6 @@ This software includes a binary and/or source version of data from
 which can be obtained from
 
   https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.0.3-20170922.tar.gz
+
+The floating point precision conversion in NumericUtils.Float16Converter is derived from work by
+Jeroen van der Zijp, granted for use under the Apache license.

java/maven.indexer/manifest.mf

@@ -3,4 +3,5 @@ OpenIDE-Module-Localizing-Bundle: org/netbeans/modules/maven/indexer/Bundle.prop
 AutoUpdate-Show-In-Client: false
 OpenIDE-Module-Specification-Version: 2.59
 OpenIDE-Module: org.netbeans.modules.maven.indexer/2
+OpenIDE-Module-Java-Dependencies: Java > 11
 

java/maven.indexer/nbproject/project.properties

@@ -17,13 +17,14 @@
 
 test.config.stableBTD.includes=**/*Test.class
 is.autoload=true
-javac.source=1.8
+javac.source=11
+javac.target=11
 javac.compilerargs=-Xlint -Xlint:-serial
-release.external/indexer-core-6.2.2.jar=modules/ext/maven/indexer-core-6.2.2.jar
-release.external/lucene-core-8.11.1.jar=modules/ext/maven/lucene-core-8.11.1.jar
-release.external/lucene-highlighter-8.11.1.jar=modules/ext/maven/lucene-highlighter-8.11.1.jar
-release.external/lucene-queryparser-8.11.1.jar=modules/ext/maven/lucene-queryparser-8.11.1.jar
-release.external/lucene-analyzers-common-8.11.1.jar=modules/ext/maven/lucene-analyzers-common-8.11.1.jar
+release.external/indexer-core-7.0.0.jar=modules/ext/maven/indexer-core-7.0.0.jar
+release.external/lucene-core-9.4.1.jar=modules/ext/maven/lucene-core-9.4.1.jar
+release.external/lucene-highlighter-9.4.1.jar=modules/ext/maven/lucene-highlighter-9.4.1.jar
+release.external/lucene-queryparser-9.4.1.jar=modules/ext/maven/lucene-queryparser-9.4.1.jar
+release.external/lucene-analysis-common-9.4.1.jar=modules/ext/maven/lucene-analysis-common-9.4.1.jar
 release.external/javax.annotation-api-1.2.jar=modules/ext/maven/javax.annotation-api-1.2.jar
 # XXX so long as Lucene is bundled here:
 sigtest.gen.fail.on.error=false

java/maven.indexer/nbproject/project.xml

@@ -175,24 +175,24 @@
                 <package>org.netbeans.modules.maven.indexer.spi.impl</package>
             </friend-packages>
             <class-path-extension>
-                <runtime-relative-path>ext/maven/indexer-core-6.2.2.jar</runtime-relative-path>
-                <binary-origin>external/indexer-core-6.2.2.jar</binary-origin>
+                <runtime-relative-path>ext/maven/indexer-core-7.0.0.jar</runtime-relative-path>
+                <binary-origin>external/indexer-core-7.0.0.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
-                <runtime-relative-path>ext/maven/lucene-core-8.11.1.jar</runtime-relative-path>
-                <binary-origin>external/lucene-core-8.11.1.jar</binary-origin>
+                <runtime-relative-path>ext/maven/lucene-core-9.4.1.jar</runtime-relative-path>
+                <binary-origin>external/lucene-core-9.4.1.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
-                <runtime-relative-path>ext/maven/lucene-highlighter-8.11.1.jar</runtime-relative-path>
-                <binary-origin>external/lucene-highlighter-8.11.1.jar</binary-origin>
+                <runtime-relative-path>ext/maven/lucene-highlighter-9.4.1.jar</runtime-relative-path>
+                <binary-origin>external/lucene-highlighter-9.4.1.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
-                <runtime-relative-path>ext/maven/lucene-queryparser-8.11.1.jar</runtime-relative-path>
-                <binary-origin>external/lucene-queryparser-8.11.1.jar</binary-origin>
+                <runtime-relative-path>ext/maven/lucene-queryparser-9.4.1.jar</runtime-relative-path>
+                <binary-origin>external/lucene-queryparser-9.4.1.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
-                <runtime-relative-path>ext/maven/lucene-analyzers-common-8.11.1.jar</runtime-relative-path>
-                <binary-origin>external/lucene-analyzers-common-8.11.1.jar</binary-origin>
+                <runtime-relative-path>ext/maven/lucene-analysis-common-9.4.1.jar</runtime-relative-path>
+                <binary-origin>external/lucene-analysis-common-9.4.1.jar</binary-origin>
             </class-path-extension>
             <class-path-extension>
                 <runtime-relative-path>ext/maven/javax.annotation-api-1.2.jar</runtime-relative-path>

java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java

@@ -61,18 +61,22 @@
 import org.apache.maven.index.updater.IndexUpdateRequest;
 import org.apache.maven.index.updater.IndexUpdater;
 import org.apache.maven.index.updater.ResourceFetcher;
-import org.apache.maven.index.updater.WagonHelper;
 import org.apache.maven.settings.Proxy;
 import org.apache.maven.settings.Server;
 import org.apache.maven.settings.crypto.DefaultSettingsDecryptionRequest;
 import org.apache.maven.settings.crypto.SettingsDecrypter;
 import org.apache.maven.settings.crypto.SettingsDecryptionResult;
+import org.apache.maven.wagon.ConnectionException;
 import org.apache.maven.wagon.ResourceDoesNotExistException;
 import org.apache.maven.wagon.Wagon;
+import org.apache.maven.wagon.WagonException;
+import org.apache.maven.wagon.authentication.AuthenticationException;
 import org.apache.maven.wagon.authentication.AuthenticationInfo;
+import org.apache.maven.wagon.authorization.AuthorizationException;
 import org.apache.maven.wagon.events.TransferListener;
 import org.apache.maven.wagon.providers.http.HttpWagon;
 import org.apache.maven.wagon.proxy.ProxyInfo;
+import org.apache.maven.wagon.repository.Repository;
 import org.codehaus.plexus.ContainerConfiguration;
 import org.codehaus.plexus.DefaultContainerConfiguration;
 import org.codehaus.plexus.DefaultPlexusContainer;
@@ -510,8 +514,13 @@ private void indexLoadedRepo(final RepositoryInfo repo, boolean updateLocal) thr
 
                     ResourceFetcher fetcher = createFetcher(wagon, listener, wagonAuth, wagonProxy);
                     listener.setFetcher(fetcher);
+
                     IndexUpdateRequest iur = new IndexUpdateRequest(indexingContext, fetcher);
-
+
+                    // Thread count for maven-indexer remote index extraction, lucene will create one additional merge
+                    // thread per extractor. 4 seems to be the sweetspot.
+                    iur.setThreads(Math.min(4, Math.max(Runtime.getRuntime().availableProcessors() - 1, 1)));
+
                     NotifyingIndexCreator nic = null;
                     for (IndexCreator ic : indexingContext.getIndexCreators()) {
                         if (ic instanceof NotifyingIndexCreator) {
@@ -1647,25 +1656,8 @@ public boolean accepts(IndexingContext ctx, ArtifactInfo ai) {
 
     }
 
-    private WagonHelper.WagonFetcher createFetcher(final Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) {
-        if(isDiag()) {
-            return new WagonHelper.WagonFetcher(wagon, listener, authenticationInfo, proxyInfo) {
-                @Override
-                public InputStream retrieve(String name) throws IOException, FileNotFoundException {
-                    String id = wagon.getRepository().getId();
-                    if(name.contains("properties") && System.getProperty("maven.diag.index.properties." + id) != null) { // NOI18N
-                        LOGGER.log(Level.INFO, "maven indexer will use local properties file: {0}", System.getProperty("maven.diag.index.properties." + id)); // NOI18N
-                        return new FileInputStream(new File(System.getProperty("maven.diag.index.properties." + id))); // NOI18N
-                    } else if(name.contains(".gz") && System.getProperty("maven.diag.index.gz." + id) != null) { // NOI18N
-                        LOGGER.log(Level.INFO, "maven indexer will use gz file: {0}", System.getProperty("maven.diag.index.gz." + id)); // NOI18N
-                        return new FileInputStream(new File(System.getProperty("maven.diag.index.gz." + id))); // NOI18N
-                    }
-                    return super.retrieve(name);
-                }
-            };
-        } else {
-            return new WagonHelper.WagonFetcher(wagon, listener, authenticationInfo, proxyInfo);
-        }
+    private ResourceFetcher createFetcher(final Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) {
+        return new WagonFetcher(wagon, listener, authenticationInfo, proxyInfo);
     }
 
     private File getIndexDirectory(final RepositoryInfo info) {
@@ -1697,4 +1689,120 @@ private void storeGroupCache(RepositoryInfo repoInfo, IndexingContext ic) throws
         }
     }
 
+    // somewhat based on maven-indexer impl (in WagonHelper) prior to removal in maven-indexer 7.0.0
+    private static class WagonFetcher implements ResourceFetcher {
+
+        private final TransferListener listener;
+        private final AuthenticationInfo authenticationInfo;
+        private final ProxyInfo proxyInfo;
+        private final Wagon wagon;
+
+        public WagonFetcher(Wagon wagon, TransferListener listener, AuthenticationInfo authenticationInfo, ProxyInfo proxyInfo) {
+            Objects.requireNonNull(wagon);
+            Objects.requireNonNull(listener);
+            this.wagon = wagon;
+            this.listener = listener;
+            this.authenticationInfo = authenticationInfo;
+            this.proxyInfo = proxyInfo;
+        }
+
+        @Override
+        public void connect(String id, String url) throws IOException {
+            Repository repository = new Repository(id, url);
+
+            try {
+                wagon.addTransferListener(listener);
+
+                if (authenticationInfo != null) {
+                    if (proxyInfo != null) {
+                        wagon.connect(repository, authenticationInfo, proxyInfo);
+                    } else {
+                        wagon.connect(repository, authenticationInfo);
+                    }
+                } else {
+                    if (proxyInfo != null) {
+                        wagon.connect(repository, proxyInfo);
+                    } else {
+                        wagon.connect(repository);
+                    }
+                }
+            } catch (AuthenticationException ex) {
+                String msg = "Authentication exception connecting to " + repository;
+                logError(msg, ex);
+                throw new IOException(msg, ex);
+            } catch (WagonException ex) {
+                String msg = "Wagon exception connecting to " + repository;
+                logError(msg, ex);
+                throw new IOException(msg, ex);
+            }
+        }
+
+        @Override
+        public void disconnect() throws IOException {
+            if (wagon != null) {
+                try {
+                    wagon.disconnect();
+                } catch (ConnectionException ex) {
+                    throw new IOException(ex.toString(), ex);
+                }
+            }
+        }
+
+        @Override
+        public InputStream retrieve(String name) throws IOException, FileNotFoundException {
+
+            if (isDiag()) {
+                String id = wagon.getRepository().getId();
+                if(name.endsWith(".properties") && System.getProperty("maven.diag.index.properties." + id) != null) { // NOI18N
+                    LOGGER.log(Level.INFO, "maven indexer will use local properties file: {0}", System.getProperty("maven.diag.index.properties." + id)); // NOI18N
+                    return new FileInputStream(new File(System.getProperty("maven.diag.index.properties." + id))); // NOI18N
+                } else if(name.endsWith(".gz") && System.getProperty("maven.diag.index.gz." + id) != null) { // NOI18N
+                    LOGGER.log(Level.INFO, "maven indexer will use gz file: {0}", System.getProperty("maven.diag.index.gz." + id)); // NOI18N
+                    return new FileInputStream(new File(System.getProperty("maven.diag.index.gz." + id))); // NOI18N
+                }
+            }
+
+            File target = Files.createTempFile(Places.getCacheDirectory().toPath(), name, "tmp").toFile();
+            target.deleteOnExit();
+
+            retrieve(name, target);
+
+            return new FileInputStream(target) {
+                @Override public void close() throws IOException {
+                    super.close();
+                    target.delete();
+                }
+            };
+        }
+
+        public void retrieve(String name, File targetFile) throws IOException, FileNotFoundException {
+            try {
+                wagon.get(name, targetFile);
+            } catch (AuthorizationException e) {
+                targetFile.delete();
+                String msg = "Authorization exception retrieving " + name;
+                logError(msg, e);
+                throw new IOException(msg, e);
+            } catch (ResourceDoesNotExistException e) {
+                targetFile.delete();
+                String msg = "Resource " + name + " does not exist";
+                logError(msg, e);
+                FileNotFoundException fileNotFoundException = new FileNotFoundException(msg);
+                fileNotFoundException.initCause(e);
+                throw fileNotFoundException;
+            } catch (WagonException e) {
+                targetFile.delete();
+                String msg = "Transfer for " + name + " failed";
+                logError(msg, e);
+                throw new IOException(msg + "; " + e.getMessage(), e);
+            }
+        }
+
+        private void logError(String msg, Exception ex) {
+            if (listener != null) {
+                listener.debug(msg + "; " + ex.getMessage());
+            }
+        }
+    }
+
 }