Skip to content

Commit

Permalink
allow limited concurrent indexing.
Browse files Browse the repository at this point in the history
Intention is to allow remote index file downloads during indexing.
  • Loading branch information
mbien committed Jan 13, 2023
1 parent 07dc395 commit 59cf3a9
Showing 1 changed file with 24 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.nio.file.StandardCopyOption;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
Expand Down Expand Up @@ -73,6 +74,7 @@
import org.apache.maven.wagon.authentication.AuthenticationException;
import org.apache.maven.wagon.authentication.AuthenticationInfo;
import org.apache.maven.wagon.authorization.AuthorizationException;
import org.apache.maven.wagon.events.TransferEvent;
import org.apache.maven.wagon.events.TransferListener;
import org.apache.maven.wagon.providers.http.HttpWagon;
import org.apache.maven.wagon.proxy.ProxyInfo;
Expand Down Expand Up @@ -149,7 +151,19 @@ public class NexusRepositoryIndexerImpl implements RepositoryIndexerImplementati
private static final HashMap<String,Mutex> repoMutexMap = new HashMap<>(4);

private static final Set<Mutex> indexingMutexes = new HashSet<>();
private static final RequestProcessor RP = new RequestProcessor("indexing", 1);

/**
* Fixed to two Threads to allow concurrent remote index downloads during local indexing
*/
private static final RequestProcessor RP = new RequestProcessor("indexing", 2);

/**
* Best effort attempt to prevent two _local_ repos to be indexed at the same time to avoid unnecessary IO contention.
* Concurrent local+remote indexing are allowed.
* This isn't perfect since a download can finish before local repos finished indexing, tests showed
* performance dosn't degrade significantly on solid state drives.
*/
private static final Semaphore localIndexingPermit = new Semaphore(1);

@Override
public boolean handlesRepository(RepositoryInfo repo) {
Expand Down Expand Up @@ -642,9 +656,18 @@ private void spawnIndexLoadedRepo(final RepositoryInfo repo) {
RP.post(() -> {
getRepoMutex(repo).writeAccess((Mutex.Action<Void>) () -> {
try {
if (repo.isLocal()) {
localIndexingPermit.acquire();
}
indexLoadedRepo(repo, true);
} catch (IOException ex) {
LOGGER.log(Level.INFO, "could not (re-)index " + repo.getId(), ex);
} catch (InterruptedException ex) {
LOGGER.log(Level.INFO, "got interrupting while waiting for indexing permit of " + repo.getId(), ex);
} finally {
if (repo.isLocal()) {
localIndexingPermit.release();
}
}
return null;
});
Expand Down

0 comments on commit 59cf3a9

Please sign in to comment.