diff --git a/java/maven.indexer/external/binaries-list b/java/maven.indexer/external/binaries-list index 79224b17ef59..02a0f1941a0f 100644 --- a/java/maven.indexer/external/binaries-list +++ b/java/maven.indexer/external/binaries-list @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. C02F9C34FBC4C698BEF9D5074966545122908D06 org.apache.maven.indexer:indexer-core:7.0.2 +20AE1ED178C812FAA78DC83C9BF218D047FD01D0 org.apache.maven.indexer:search-api:7.0.2 +5967AEFBE9281C46155E73BBFC1E2BC4054CC92F org.apache.maven.indexer:search-backend-smo:7.0.2 2C08C7A491E9D033BB4806E0A45496E3A0667217 org.apache.lucene:lucene-core:9.6.0 A4819CA127C46A2759FDF091F41512C56C441FA7 org.apache.lucene:lucene-backward-codecs:9.6.0 B9E14451C73FA0BC8E71A2CFE12A442F37A53C69 org.apache.lucene:lucene-highlighter:9.6.0 C2F9EF9B7336981495272E99A1DF678A97966102 org.apache.lucene:lucene-queryparser:9.6.0 17A8B808BB7BF5F49FBFB9CBFF821433E2908E22 org.apache.lucene:lucene-analysis-common:9.6.0 479C1E06DB31C432330183F5CAE684163F186146 javax.annotation:javax.annotation-api:1.2 +B3ADD478D4382B78EA20B1671390A858002FEB6C com.google.code.gson:gson:2.10.1 diff --git a/java/maven.indexer/nbproject/project.properties b/java/maven.indexer/nbproject/project.properties index 500ec728c322..fda5aa1a253a 100644 --- a/java/maven.indexer/nbproject/project.properties +++ b/java/maven.indexer/nbproject/project.properties @@ -21,6 +21,9 @@ javac.source=11 javac.target=11 javac.compilerargs=-Xlint -Xlint:-serial release.external/indexer-core-7.0.2.jar=modules/ext/maven/indexer-core-7.0.2.jar +release.external/search-api-7.0.2.jar=modules/ext/maven/search-api-7.0.2.jar +release.external/search-backend-smo-7.0.2.jar=modules/ext/maven/search-backend-smo-7.0.2.jar +release.external/gson-2.10.1.jar=modules/ext/maven/gson-2.10.1.jar release.external/lucene-core-9.6.0.jar=modules/ext/maven/lucene-core-9.6.0.jar release.external/lucene-backward-codecs-9.6.0.jar=modules/ext/maven/lucene-backward-codecs-9.6.0.jar release.external/lucene-highlighter-9.6.0.jar=modules/ext/maven/lucene-highlighter-9.6.0.jar diff --git a/java/maven.indexer/nbproject/project.xml b/java/maven.indexer/nbproject/project.xml index fb0e05a8a539..fc2f00355b90 100644 --- a/java/maven.indexer/nbproject/project.xml +++ b/java/maven.indexer/nbproject/project.xml @@ -178,6 +178,14 @@ ext/maven/indexer-core-7.0.2.jar external/indexer-core-7.0.2.jar + + ext/maven/search-api-7.0.2.jar + external/search-api-7.0.2.jar + + + ext/maven/search-backend-smo-7.0.2.jar + external/search-backend-smo-7.0.2.jar + ext/maven/lucene-core-9.6.0.jar external/lucene-core-9.6.0.jar @@ -202,6 +210,10 @@ ext/maven/javax.annotation-api-1.2.jar external/javax.annotation-api-1.2.jar + + ext/maven/gson-2.10.1.jar + external/gson-2.10.1.jar + diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java index 5d1fb15450ef..7c59e3bde71f 100644 --- a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/NexusRepositoryIndexerImpl.java @@ -34,9 +34,12 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -65,6 +68,11 @@ import org.apache.maven.index.updater.IndexUpdateRequest; import org.apache.maven.index.updater.IndexUpdater; import org.apache.maven.index.updater.ResourceFetcher; +import org.apache.maven.search.SearchRequest; +import org.apache.maven.search.backend.smo.SmoSearchBackend; +import org.apache.maven.search.backend.smo.SmoSearchBackendFactory; +import org.apache.maven.search.request.FieldQuery; +import org.apache.maven.search.request.Paging; import org.apache.maven.settings.Proxy; import org.apache.maven.settings.Server; import org.apache.maven.settings.crypto.DefaultSettingsDecryptionRequest; @@ -167,6 +175,8 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati * For remote repo download and indexing tasks. */ private static final RequestProcessor RP_REMOTE = new RequestProcessor("maven-remote-indexing"); + + private final SmoSearchBackend smo = new SmoSearchBackendFactory().createDefault(); @Override public boolean handlesRepository(RepositoryInfo repo) { @@ -440,7 +450,7 @@ private boolean loadIndexingContext(final RepositoryInfo info) throws IOExceptio try { IndexSearcher.setMaxClauseCount(max); response = searcher.searchIteratorPaged(isr, contexts); - LOGGER.log(Level.FINE, "passed on {0} clauses processing {1} with {2} hits", new Object[] {max, q, response.getTotalHitsCount()}); + LOGGER.log(Level.INFO, "passed on {0} clauses processing {1} with {2} hits", new Object[] {max, q, response.getTotalHitsCount()}); return response; } catch (IndexSearcher.TooManyClauses exc) { LOGGER.log(Level.FINE, "TooManyClauses on {0} clauses processing {1}", new Object[] {max, q}); @@ -1196,12 +1206,38 @@ private ResultImplementation getVersions(final String groupId, fi @Override public ResultImplementation findVersionsByClass(final String className, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findVersionsByClass(className, result1, result1.getSkipped(), false); - }); - return findVersionsByClass(className, result, repos, true); + + Optional central = repos.stream() + .filter(repo -> repo.getId().equals(smo.getRepositoryId())) + .findFirst(); + + // remote index contains no class data -> use web service + if (central.isPresent()) { + List otherRepos = new ArrayList<>(repos); + otherRepos.remove(central.get()); + + SearchRequest request = new SearchRequest(new Paging(64), + FieldQuery.fieldQuery(className.contains(".") ? + org.apache.maven.search.MAVEN.FQ_CLASS_NAME + : org.apache.maven.search.MAVEN.CLASS_NAME, className)); + + return mergeResults( + () -> (ResultImpl) findVersionsByClass(className, otherRepos), + () -> { + try { + return new SMOResultImpl(smo.getRepositoryId(), smo.search(request)); + } catch (IOException ex) { + return SMOResultImpl.empty(); + } + }); + } else { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findVersionsByClass(className, result1, result1.getSkipped(), false); + }); + return findVersionsByClass(className, result, repos, true); + } } - + private ResultImplementation findVersionsByClass(final String className, final ResultImpl result, List repos, final boolean skipUnIndexed) { final List infos = new ArrayList<>(result.getResults()); final SkippedAction skipAction = new SkippedAction(result); @@ -1345,14 +1381,36 @@ private static void convertToNBGroupInfo(Collection artifactInfos } } - - @Override public ResultImplementation findBySHA1(final String sha1, List repos) { - ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { - findBySHA1(sha1, result1, result1.getSkipped(), false); - }); - return findBySHA1(sha1, result, repos, true); + + Optional central = repos.stream() + .filter(repo -> repo.getId().equals(smo.getRepositoryId())) + .findFirst(); + + // remote index contains no sh1 data -> use web service + if (central.isPresent()) { + List otherRepos = new ArrayList<>(repos); + otherRepos.remove(central.get()); + + SearchRequest request = new SearchRequest(new Paging(8), + FieldQuery.fieldQuery(org.apache.maven.search.MAVEN.SHA1, sha1)); + + return mergeResults( + () -> (ResultImpl) findBySHA1(sha1, otherRepos), + () -> { + try { + return new SMOResultImpl(smo.getRepositoryId(), smo.search(request)); + } catch (IOException ex) { + return SMOResultImpl.empty(); + } + }); + } else { + ResultImpl result = new ResultImpl<>((ResultImpl result1) -> { + findBySHA1(sha1, result1, result1.getSkipped(), false); + }); + return findBySHA1(sha1, result, repos, true); + } } private ResultImplementation findBySHA1(final String sha1, final ResultImpl result, List repos, final boolean skipUnIndexed) { @@ -1699,6 +1757,24 @@ static NBVersionInfo convertToNBVersionInfo(ArtifactInfo ai) { return nbvi; } + private static ResultImpl mergeResults(Supplier> mergedResultsSupplier, Supplier> resultsToAddSupplier) { + + Instant start = Instant.now(); + ResultImpl mergedResults = mergedResultsSupplier.get(); + List r1 = mergedResults.getResults(); + List r2 = resultsToAddSupplier.get().getResults(); + LOGGER.log(Level.INFO, "response time: "+Duration.between(start, Instant.now()).toMillis()); + + List merged = new ArrayList<>(r2.size()+r1.size()); + merged.addAll(r1); + merged.addAll(r2); + + mergedResults.setResults(merged); + mergedResults.addReturnedResultCount(r2.size()); + mergedResults.addTotalResultCount(r2.size()); + return mergedResults; + } + private static Query setBooleanRewrite (final Query q) { if (q instanceof MultiTermQuery) { ((MultiTermQuery)q).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); diff --git a/java/maven.indexer/src/org/netbeans/modules/maven/indexer/SMOResultImpl.java b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/SMOResultImpl.java new file mode 100644 index 000000000000..1f103a2bdb68 --- /dev/null +++ b/java/maven.indexer/src/org/netbeans/modules/maven/indexer/SMOResultImpl.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.maven.indexer; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.maven.search.backend.smo.SmoSearchResponse; +import org.netbeans.modules.maven.indexer.api.NBVersionInfo; +import org.netbeans.modules.maven.indexer.spi.ResultImplementation; + +/** + * Wraps search-maven-org query results. + * + * @author mbien + */ +final class SMOResultImpl implements ResultImplementation { + + private List list = null; + private final String repoId; + private final SmoSearchResponse response; + + public SMOResultImpl(String repoId, SmoSearchResponse response) { + this.repoId = repoId; + this.response = response; + } + + @Override + public List getResults() { + if (list == null) { + list = response.getPage().stream() + .map(rec -> new NBVersionInfo( + repoId, + rec.getValue(org.apache.maven.search.MAVEN.GROUP_ID), + rec.getValue(org.apache.maven.search.MAVEN.ARTIFACT_ID), + rec.getValue(org.apache.maven.search.MAVEN.VERSION), + rec.getValue(org.apache.maven.search.MAVEN.PACKAGING), // todo, type is used in the UI as packaging?? + rec.getValue(org.apache.maven.search.MAVEN.PACKAGING), + null, + null, + rec.getValue(org.apache.maven.search.MAVEN.CLASSIFIER))) + .collect(Collectors.toList()); + } + return list; + } + + @Override + public boolean isPartial() { + return false; + } + + @Override + public void waitForSkipped() { + throw new UnsupportedOperationException(); + } + + @Override + public int getTotalResultCount() { + return response.getTotalHits(); + } + + @Override + public int getReturnedResultCount() { + return response.getCurrentHits(); + } + + public static ResultImplementation empty() { + return new ResultImplementation() { + @Override + public boolean isPartial() { + return false; + } + @Override + public List getResults() { + return Collections.emptyList(); + } + @Override + public int getTotalResultCount() { + return 0; + } + @Override + public int getReturnedResultCount() { + return 0; + } + @Override + public void waitForSkipped() {} + }; + } + +}