Skip to content

Commit

Permalink
Add support for new repository target location type
Browse files Browse the repository at this point in the history
Fix #2838
  • Loading branch information
laeubi committed Sep 21, 2023
1 parent 1500f0c commit 3f60150
Show file tree
Hide file tree
Showing 11 changed files with 548 additions and 224 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import java.io.OutputStream;
import java.net.URI;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.NumberFormat;
import java.util.Map;
import java.util.concurrent.Executor;
Expand All @@ -44,25 +46,27 @@

@Component(role = org.eclipse.equinox.internal.p2.repository.Transport.class, hint = "tycho")
public class TychoRepositoryTransport extends org.eclipse.equinox.internal.p2.repository.Transport
implements IAgentServiceFactory {
implements IAgentServiceFactory {

private static final int MAX_DOWNLOAD_THREADS = Integer.getInteger("tycho.p2.transport.max-download-threads", 4);

private static final boolean DEBUG_REQUESTS = Boolean.getBoolean("tycho.p2.transport.debug");

private static final Executor DOWNLOAD_EXECUTOR = Executors.newFixedThreadPool(MAX_DOWNLOAD_THREADS, new ThreadFactory() {

private AtomicInteger cnt = new AtomicInteger();
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("Tycho-Download-Thread-" + cnt.getAndIncrement());
thread.setDaemon(true);
return thread;
}
});
private static final Executor DOWNLOAD_EXECUTOR = Executors.newFixedThreadPool(MAX_DOWNLOAD_THREADS,
new ThreadFactory() {

private AtomicInteger cnt = new AtomicInteger();

private NumberFormat numberFormat = NumberFormat.getNumberInstance();
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r);
thread.setName("Tycho-Download-Thread-" + cnt.getAndIncrement());
thread.setDaemon(true);
return thread;
}
});

private NumberFormat numberFormat = NumberFormat.getNumberInstance();

@Requirement
Logger logger;
Expand All @@ -73,24 +77,24 @@ public Thread newThread(Runnable r) {
@Requirement(role = TransportProtocolHandler.class)
Map<String, TransportProtocolHandler> transportProtocolHandlers;

private LongAdder requests = new LongAdder();
private LongAdder indexRequests = new LongAdder();
private LongAdder requests = new LongAdder();
private LongAdder indexRequests = new LongAdder();

public TychoRepositoryTransport() {
numberFormat.setMaximumFractionDigits(2);
}

@Override
public IStatus download(URI toDownload, OutputStream target, long startPos, IProgressMonitor monitor) {
if (startPos > 0) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"range downloads are not implemented");
}
return download(toDownload, target, monitor);
}

@Override
public IStatus download(URI toDownload, OutputStream target, IProgressMonitor monitor) {
numberFormat.setMaximumFractionDigits(2);
}

@Override
public IStatus download(URI toDownload, OutputStream target, long startPos, IProgressMonitor monitor) {
if (startPos > 0) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"range downloads are not implemented");
}
return download(toDownload, target, monitor);
}

@Override
public IStatus download(URI toDownload, OutputStream target, IProgressMonitor monitor) {
String id = "p2"; // TODO we might compute the id from the IRepositoryIdManager based on the URI?
if (cacheConfig.isInteractive()) {
logger.info("Downloading from " + id + ": " + toDownload);
Expand All @@ -101,41 +105,40 @@ public IStatus download(URI toDownload, OutputStream target, IProgressMonitor mo
stream(toDownload, monitor).transferTo(statusOutputStream);
DownloadStatus downloadStatus = statusOutputStream.getStatus();
if (cacheConfig.isInteractive()) {
logger.info(
"Downloaded from " + id + ": " + toDownload + " ("
+ FileUtils.byteCountToDisplaySize(downloadStatus.getFileSize())
+ " at " + FileUtils.byteCountToDisplaySize(downloadStatus.getTransferRate()) + "/s)");
logger.info("Downloaded from " + id + ": " + toDownload + " ("
+ FileUtils.byteCountToDisplaySize(downloadStatus.getFileSize()) + " at "
+ FileUtils.byteCountToDisplaySize(downloadStatus.getTransferRate()) + "/s)");
}
return reportStatus(downloadStatus, target);
} catch (AuthenticationFailedException e) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"authentication failed for " + toDownload, e);
} catch (IOException e) {
return reportStatus(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e), target);
} catch (CoreException e) {
return reportStatus(e.getStatus(), target);
}
}
} catch (AuthenticationFailedException e) {
return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"authentication failed for " + toDownload, e);
} catch (IOException e) {
return reportStatus(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e), target);
} catch (CoreException e) {
return reportStatus(e.getStatus(), target);
}
}

private IStatus reportStatus(IStatus status, OutputStream target) {
if (target instanceof IStateful stateful) {
stateful.setStatus(status);
}
return status;
}
if (target instanceof IStateful stateful) {
stateful.setStatus(status);
}
return status;
}

@Override
@Override
public InputStream stream(URI toDownload, IProgressMonitor monitor)
throws FileNotFoundException, CoreException, AuthenticationFailedException {
if (DEBUG_REQUESTS) {
logger.debug("Request stream for " + toDownload);
logger.debug("Request stream for " + toDownload);
}
requests.increment();
if (toDownload.toASCIIString().endsWith("p2.index")) {
indexRequests.increment();
}
try {
requests.increment();
if (toDownload.toASCIIString().endsWith("p2.index")) {
indexRequests.increment();
}
try {
TransportProtocolHandler handler = getHandler(toDownload);
if (handler != null) {
File cachedFile = handler.getFile(toDownload);
Expand All @@ -146,28 +149,28 @@ public InputStream stream(URI toDownload, IProgressMonitor monitor)
return new FileInputStream(cachedFile);
}
}
return toDownload.toURL().openStream();
} catch (FileNotFoundException e) {
return toDownload.toURL().openStream();
} catch (FileNotFoundException e) {
if (DEBUG_REQUESTS) {
logger.debug(" --> not found!");
}
throw e;
} catch (IOException e) {
logger.debug(" --> not found!");
}
throw e;
} catch (IOException e) {
if (e instanceof AuthenticationFailedException afe) {
throw afe;
}
if (DEBUG_REQUESTS) {
logger.debug(" --> generic error: " + e);
}
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
} finally {
logger.debug(" --> generic error: " + e);
}
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
} finally {
if (DEBUG_REQUESTS) {
logger.debug("Total number of requests: " + requests.longValue() + " (" + indexRequests.longValue()
+ " for p2.index)");
}
}
}
logger.debug("Total number of requests: " + requests.longValue() + " (" + indexRequests.longValue()
+ " for p2.index)");
}
}
}

TransportProtocolHandler getHandler(URI uri) {
String scheme = uri.getScheme();
Expand All @@ -181,9 +184,9 @@ TransportProtocolHandler getHandler(URI uri) {
return null;
}

@Override
public long getLastModified(URI toDownload, IProgressMonitor monitor)
throws CoreException, FileNotFoundException, AuthenticationFailedException {
@Override
public long getLastModified(URI toDownload, IProgressMonitor monitor)
throws CoreException, FileNotFoundException, AuthenticationFailedException {
try {
TransportProtocolHandler handler = getHandler(toDownload);
if (handler != null) {
Expand All @@ -199,14 +202,33 @@ public long getLastModified(URI toDownload, IProgressMonitor monitor)
throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(),
"download from " + toDownload + " failed", e));
}
}
@Override
public Object createService(IProvisioningAgent agent) {
return this;
}
}

@Override
public Object createService(IProvisioningAgent agent) {
return this;
}

public static Executor getDownloadExecutor() {
return DOWNLOAD_EXECUTOR;
}

public File downloadToFile(URI uri) throws IOException {
TransportProtocolHandler handler = getHandler(uri);
if (handler != null) {
File file = handler.getFile(uri);
if (file != null) {
return file;
}
}
Path tempFile = Files.createTempFile("tycho", ".tmp");
tempFile.toFile().deleteOnExit();
try {
Files.copy(stream(uri, null), tempFile);
return tempFile.toFile();
} catch (CoreException e) {
throw new IOException(e);
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*******************************************************************************
* Copyright (c) 2023 Christoph Läubrich and others.
*
* This program and the accompanying materials
* are made available under the terms of the Eclipse Public License 2.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Christoph Läubrich - initial API and implementation
*******************************************************************************/
package org.eclipse.tycho.p2resolver;

import java.io.File;
import java.io.InputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;

import org.apache.commons.io.FilenameUtils;
import org.eclipse.equinox.internal.p2.publisher.eclipse.FeatureParser;
import org.eclipse.equinox.internal.p2.repository.Transport;
import org.eclipse.equinox.p2.core.IProvisioningAgent;
import org.eclipse.equinox.p2.metadata.IArtifactKey;
import org.eclipse.equinox.p2.metadata.IInstallableUnit;
import org.eclipse.equinox.p2.publisher.IPublisherInfo;
import org.eclipse.equinox.p2.publisher.PublisherInfo;
import org.eclipse.equinox.p2.publisher.eclipse.BundlesAction;
import org.eclipse.equinox.p2.publisher.eclipse.Feature;
import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor;
import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
import org.eclipse.osgi.service.resolver.BundleDescription;
import org.eclipse.tycho.core.resolver.target.FileArtifactRepository;
import org.eclipse.tycho.core.resolver.target.SupplierMetadataRepository;
import org.eclipse.tycho.core.shared.MavenLogger;
import org.eclipse.tycho.p2.resolver.BundlePublisher;
import org.eclipse.tycho.p2.resolver.FeaturePublisher;
import org.eclipse.tycho.p2maven.transport.TychoRepositoryTransport;
import org.eclipse.tycho.targetplatform.TargetDefinitionContent;
import org.eclipse.tycho.targetplatform.TargetDefinitionResolutionException;
import org.osgi.resource.Capability;
import org.osgi.resource.Requirement;

import aQute.bnd.osgi.repository.ResourcesRepository;
import aQute.bnd.osgi.repository.XMLResourceParser;
import aQute.bnd.osgi.resource.ResourceUtils;
import aQute.bnd.osgi.resource.ResourceUtils.ContentCapability;

public class RepositoryLocationContent implements TargetDefinitionContent {
private final Map<IArtifactDescriptor, IInstallableUnit> repositoryContent = new HashMap<>();
private SupplierMetadataRepository metadataRepository;
private FileArtifactRepository artifactRepository;

public RepositoryLocationContent(URI uri, Collection<Requirement> requirements, IProvisioningAgent agent,
MavenLogger logger) throws TargetDefinitionResolutionException {
TychoRepositoryTransport tychoTransport = (TychoRepositoryTransport) agent.getService(Transport.class);

metadataRepository = new SupplierMetadataRepository(agent, () -> repositoryContent.values().iterator());
metadataRepository.setLocation(uri);
metadataRepository.setName(String.valueOf(uri));
artifactRepository = new FileArtifactRepository(agent, () -> repositoryContent.keySet().stream()
.filter(Predicate.not(FeaturePublisher::isMetadataOnly)).iterator());
artifactRepository.setName(String.valueOf(uri));
artifactRepository.setLocation(uri);
List<Feature> features = new ArrayList<>();
ResourcesRepository repository;
try (InputStream stream = tychoTransport.stream(uri, null)) {
repository = new ResourcesRepository(XMLResourceParser.getResources(stream, uri));
} catch (Exception e) {
throw new TargetDefinitionResolutionException("Can't load the repository from URI " + uri, e);
}
Map<Requirement, Collection<Capability>> providers = repository.findProviders(requirements);
//TODO once we have changed Tycho to use resources this can be optimized to not download all selected content here ...
List<ContentCapability> contentCapabilities = providers.values().stream().flatMap(Collection::stream)
.map(Capability::getResource).distinct().map(ResourceUtils::getContentCapability)
.filter(Objects::nonNull).toList();
for (ContentCapability content : contentCapabilities) {
URI url = content.url();
logger.info("Loading " + url + "...");
try {
File file = tychoTransport.downloadToFile(url);
if (!"jar".equalsIgnoreCase(FilenameUtils.getExtension(file.getName()))) {
logger.info("Skip non-jar artifact (" + file + ")");
continue;
}
Feature feature = new FeatureParser().parse(file);
if (feature != null) {
feature.setLocation(file.getAbsolutePath());
features.add(feature);
continue;
}
BundleDescription bundleDescription = BundlesAction.createBundleDescription(file);
if (bundleDescription == null || bundleDescription.getSymbolicName() == null) {
continue;
}
publish(bundleDescription, file);
} catch (Exception e) {
throw new TargetDefinitionResolutionException("Can't fetch resource from " + url, e);
}
}
FeaturePublisher.publishFeatures(features, repositoryContent::put, logger);
}

private void publish(BundleDescription bundleDescription, File bundleLocation) {
IArtifactKey key = BundlesAction.createBundleArtifactKey(bundleDescription.getSymbolicName(),
bundleDescription.getVersion().toString());
IArtifactDescriptor descriptor = FileArtifactRepository.forFile(bundleLocation, key);
PublisherInfo publisherInfo = new PublisherInfo();
publisherInfo.setArtifactOptions(IPublisherInfo.A_INDEX);
IInstallableUnit iu = BundlePublisher.publishBundle(bundleDescription, descriptor, publisherInfo);
repositoryContent.put(descriptor, iu);
}

@Override
public IMetadataRepository getMetadataRepository() {
return metadataRepository;
}

@Override
public IArtifactRepository getArtifactRepository() {
return artifactRepository;
}

}
Loading

0 comments on commit 3f60150

Please sign in to comment.