From b1b992c877026f9d0903f58011835d7dc4996559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Wed, 23 Mar 2022 05:53:46 +0100 Subject: [PATCH] Issue #797 - add a custom tycho transport --- RELEASE_NOTES.md | 25 + .../shared/MavenRepositoryLocation.java | 3 +- .../shared/MavenRepositorySettings.java | 13 +- .../tycho/core/shared/MavenContext.java | 15 +- .../tycho/core/shared/MockMavenContext.java | 12 + .../local/GAVArtifactDescriptorTest.java | 6 +- .../manager/ReactorRepositoryManagerTest.java | 2 +- .../remote/RemoteAgentMavenMirrorsTest.java | 9 +- .../META-INF/MANIFEST.MF | 1 + .../OSGI-INF/remoteAgentManager.xml | 1 + .../eclipse/tycho/p2/remote/CacheEntry.java | 28 + .../eclipse/tycho/p2/remote/RemoteAgent.java | 73 ++- .../tycho/p2/remote/RemoteAgentManager.java | 9 +- .../remote/RemoteRepositoryLoadingHelper.java | 13 +- .../p2/remote/SharedHttpCacheStorage.java | 592 ++++++++++++++++++ .../p2/remote/TychoRepositoryTransport.java | 194 ++++++ .../TychoRepositoryTransportCacheManager.java | 77 +++ .../META-INF/MANIFEST.MF | 3 +- .../util/MavenServiceStubbingTestBase.java | 10 +- .../MavenContextConfigurator.java | 33 +- .../RepositorySettingsConfigurator.java | 5 +- 21 files changed, 1086 insertions(+), 38 deletions(-) create mode 100644 tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/CacheEntry.java create mode 100644 tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/SharedHttpCacheStorage.java create mode 100644 tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransport.java create mode 100644 tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransportCacheManager.java diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 088608f668..5f39d8aa6a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,31 @@ This page describes the noteworthy improvements provided by each release of Ecli ## 3.0.0 (under development) +### Improved P2 transport for more efficiently http-cache handling and improved offline mode + +P2 default transport is more designed as a weak cache that assumes the user is always online. +While for an IDE that might be sufficient as target resolution is only performed once in a while and updates are triggered by explicit user request, for tycho this does not work well: + +- Builds are often trigger on each code change, requiring repeated target resolution +- Builds might be asked to run in an offline mode +- If there is a temporary server outrage one might to fallback to the previous state for this build instead of fail completely +- Build times are often a rare resource one don't want to waste waiting for servers, bandwidth might even be limited or you have to pay for it + +Because of this, Tycho now includes a brand new caching P2 transport that allows advanced aching, offline handling and fallback to cache in case of server failures. The transport is enabled by default so nothing has to be done, just in case you want the old behavior you can set `-D=tycho.p2.transport=ecf` beside that the following properties might be interesting: + +#### Force cache-revalidation +If you run maven with the `-U` switch Tycho revalidates the cache, this is useful if you have changed an updatesite in an unusual way (e.g. adding new index files) as tycho now also caches not found items to speed-up certain scenarios where many non existing files are queried. + +#### Configure minimum caching age + +Some servers don't provide you with sufficient caching information, for this purpose, tychy by default assumes a minimum caching age of one hour. You can switch this off, or configure a longer delay by using `-Dtycho.p2.transport.min-cache-minutes=`. +Choosing a sensible value could greatly improve your build times and lower banYdwith usage. If your build contains a mixture of released and 'snapshot' sites you have the following options: + +1. Consider adding a mirror to your settings.xml for the snapshot page that point to a file-local copy (e.g. output of another build) +2. Configure the webserver of your snapshot site with the `Cache-Control: must-revalidate` header in which case tycho ignores any minimum age +3. Use `-Dtycho.p2.transport.min-cache-minutes=0` this will still improve the time to resolve the target + + ### Automatic generation of PDE source bundles for pom-first bundles PDE requires some special headers to detect a bundle as a "Source Bundle", there is now a new mojo `tycho-source-plugin:generate-pde-source-header` that support this, it requires the following configuration: diff --git a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositoryLocation.java b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositoryLocation.java index eaf6011fc5..5dfcfc9644 100644 --- a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositoryLocation.java +++ b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositoryLocation.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012 SAP AG and others. + * Copyright (c) 2012, 2022 SAP AG and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -9,6 +9,7 @@ * * Contributors: * SAP AG - initial API and implementation + * Christoph Läubrich - Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.core.resolver.shared; diff --git a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositorySettings.java b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositorySettings.java index d093b4ce1f..06e378684e 100644 --- a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositorySettings.java +++ b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositorySettings.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012, 2013 SAP AG and others. + * Copyright (c) 2012, 2022 SAP AG and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -9,9 +9,12 @@ * * Contributors: * SAP AG - initial API and implementation + * Christoph Läubrich - Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.core.resolver.shared; +import java.net.URI; + /** * Provides the mirror configuration and credentials from the Maven settings for loading remote p2 * repositories. @@ -21,10 +24,12 @@ public interface MavenRepositorySettings { public final class Credentials { private final String userName; private final String password; + private final URI url; - public Credentials(String userName, String password) { + public Credentials(String userName, String password, URI uri) { this.userName = userName; this.password = password; + this.url = uri; } public String getUserName() { @@ -34,6 +39,10 @@ public String getUserName() { public String getPassword() { return password; } + + public URI getURI() { + return url; + } } /** diff --git a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MavenContext.java b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MavenContext.java index 54dca32afe..f36967966d 100644 --- a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MavenContext.java +++ b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MavenContext.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2011, 2020 SAP AG and others. + * Copyright (c) 2011, 2022 SAP AG and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -16,8 +16,10 @@ import java.io.File; import java.util.Collection; import java.util.Properties; +import java.util.stream.Stream; import org.eclipse.tycho.ReactorProject; +import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation; /** * Makes maven information which is constant for the whole maven session available as a service to @@ -34,6 +36,11 @@ public interface MavenContext { */ public boolean isOffline(); + /** + * whether maven was started with the update-snapshots mode (CLI option "-U") + */ + boolean isUpdateSnapshots(); + /** * Session-global properties merged from (in order of precedence) *
    @@ -59,4 +66,10 @@ public interface MavenContext { */ public String getExtension(String artifactType); + /** + * + * @return a collection of all {@link MavenRepositoryLocation}s know to the maven context + */ + Stream getMavenRepositoryLocations(); + } diff --git a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MockMavenContext.java b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MockMavenContext.java index 8c47279da0..c10f6c3e8e 100644 --- a/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MockMavenContext.java +++ b/tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MockMavenContext.java @@ -14,9 +14,11 @@ import java.io.File; import java.util.Properties; +import java.util.stream.Stream; import org.eclipse.tycho.ArtifactType; import org.eclipse.tycho.PackagingType; +import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation; public class MockMavenContext extends MavenContextImpl { @@ -59,4 +61,14 @@ public String getExtension(String artifactType) { } } + @Override + public boolean isUpdateSnapshots() { + return false; + } + + @Override + public Stream getMavenRepositoryLocations() { + return Stream.empty(); + } + } diff --git a/tycho-bundles/org.eclipse.tycho.p2.maven.repository.tests/src/test/java/org/eclipse/tycho/repository/local/GAVArtifactDescriptorTest.java b/tycho-bundles/org.eclipse.tycho.p2.maven.repository.tests/src/test/java/org/eclipse/tycho/repository/local/GAVArtifactDescriptorTest.java index 26cd69925b..a5bdbf1407 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.maven.repository.tests/src/test/java/org/eclipse/tycho/repository/local/GAVArtifactDescriptorTest.java +++ b/tycho-bundles/org.eclipse.tycho.p2.maven.repository.tests/src/test/java/org/eclipse/tycho/repository/local/GAVArtifactDescriptorTest.java @@ -21,7 +21,7 @@ import org.eclipse.equinox.p2.metadata.IArtifactKey; import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.repository.artifact.spi.ArtifactDescriptor; -import org.eclipse.tycho.core.shared.MavenContextImpl; +import org.eclipse.tycho.core.shared.MockMavenContext; import org.eclipse.tycho.p2.repository.GAV; import org.eclipse.tycho.p2.repository.MavenRepositoryCoordinates; import org.junit.Test; @@ -170,7 +170,7 @@ public void testGetLocalRepositoryPath() { OTHER_EXTENSION); subject = new GAVArtifactDescriptor(createP2Descriptor(), coordinates); - assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MavenContextImpl(null, false, null, null) { + assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MockMavenContext(null, false, null, null) { @Override public String getExtension(String artifactType) { @@ -185,7 +185,7 @@ public void testGetLocalRepositoryPathWithDefaults() { DEFAULT_EXTENSION); subject = new GAVArtifactDescriptor(createP2Descriptor(), coordinates); - assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MavenContextImpl(null, false, null, null) { + assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MockMavenContext(null, false, null, null) { @Override public String getExtension(String artifactType) { diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/manager/ReactorRepositoryManagerTest.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/manager/ReactorRepositoryManagerTest.java index 2006d89f92..04c0ca3a69 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/manager/ReactorRepositoryManagerTest.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/manager/ReactorRepositoryManagerTest.java @@ -77,7 +77,7 @@ public void testReactorRepositoryManagerFacadeServiceAvailability() throws Excep @Test public void testTargetPlatformComputationInIntegration() throws Exception { subject = getService(ReactorRepositoryManagerFacade.class); - + assertThat(subject, is(notNullValue())); ReactorProject currentProject = new ReactorProjectStub("reactor-artifact"); TargetPlatformConfigurationStub tpConfig = new TargetPlatformConfigurationStub(); diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/remote/RemoteAgentMavenMirrorsTest.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/remote/RemoteAgentMavenMirrorsTest.java index 0d8abb9f33..346a571958 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/remote/RemoteAgentMavenMirrorsTest.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/remote/RemoteAgentMavenMirrorsTest.java @@ -54,10 +54,15 @@ public class RemoteAgentMavenMirrorsTest { public void initSubject() throws Exception { File localRepository = tempManager.newFolder("localRepo"); MavenContext mavenContext = new MockMavenContext(localRepository, OFFLINE, logVerifier.getLogger(), - new Properties()); + new Properties()) { + @Override + public boolean isUpdateSnapshots() { + return true; + } + }; mavenRepositorySettings = new MavenRepositorySettingsStub(); - subject = new RemoteAgent(mavenContext, mavenRepositorySettings, OFFLINE); + subject = new RemoteAgent(mavenContext, null, mavenRepositorySettings, OFFLINE); } @Test diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/META-INF/MANIFEST.MF b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/META-INF/MANIFEST.MF index 03079b7c47..e7b3ba8ea1 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/META-INF/MANIFEST.MF +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/META-INF/MANIFEST.MF @@ -33,6 +33,7 @@ Export-Package: org.eclipse.tycho.p2.impl;x-friends:="org.eclipse.tycho.p2.impl. org.eclipse.tycho.p2.target.ee;x-friends:="org.eclipse.tycho.p2.tools.impl", org.eclipse.tycho.p2.util.resolution Import-Package: org.apache.commons.io;version="2.8.0", + org.eclipse.ecf.provider.filetransfer.util;version="3.2.0", org.eclipse.tycho, org.eclipse.tycho.artifacts, org.eclipse.tycho.core.ee.shared, diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/OSGI-INF/remoteAgentManager.xml b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/OSGI-INF/remoteAgentManager.xml index ec6e39038c..49c68360e5 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/OSGI-INF/remoteAgentManager.xml +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/OSGI-INF/remoteAgentManager.xml @@ -6,4 +6,5 @@ + diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/CacheEntry.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/CacheEntry.java new file mode 100644 index 0000000000..4dfc01b20d --- /dev/null +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/CacheEntry.java @@ -0,0 +1,28 @@ +/******************************************************************************* + * Copyright (c) 2022 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2.remote; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.function.Function; + +import org.eclipse.core.net.proxy.IProxyService; +import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings.Credentials; + +public interface CacheEntry { + + long getLastModified(IProxyService proxyService, Function credentialsProvider) throws IOException; + + File getCacheFile(IProxyService proxyService, Function credentialsProvider) throws IOException; +} diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgent.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgent.java index b102540c90..17fa140b46 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgent.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgent.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012 SAP AG and others. + * Copyright (c) 2012, 2022 SAP AG and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -9,15 +9,21 @@ * * Contributors: * SAP AG - initial API and implementation + * Christoph Läubrich - Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.p2.remote; +import java.util.Objects; +import java.util.stream.Stream; + +import org.eclipse.core.net.proxy.IProxyService; import org.eclipse.equinox.internal.p2.repository.CacheManager; import org.eclipse.equinox.internal.p2.repository.Transport; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager; import org.eclipse.equinox.p2.repository.metadata.IMetadataRepositoryManager; +import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation; import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings; import org.eclipse.tycho.core.shared.MavenContext; import org.eclipse.tycho.core.shared.MavenLogger; @@ -28,39 +34,70 @@ public class RemoteAgent implements IProvisioningAgent { private IProvisioningAgent delegate; - public RemoteAgent(MavenContext mavenContext, MavenRepositorySettings mavenRepositorySettings, - boolean disableMirrors) throws ProvisionException { - this.delegate = createConfiguredProvisioningAgent(mavenContext, disableMirrors, mavenRepositorySettings); + public RemoteAgent(MavenContext mavenContext, IProxyService proxyService, + MavenRepositorySettings mavenRepositorySettings, boolean disableMirrors) throws ProvisionException { + this.delegate = createConfiguredProvisioningAgent(mavenContext, proxyService, disableMirrors, + mavenRepositorySettings); } // constructor for tests RemoteAgent(MavenContext mavenContext, boolean disableP2Mirrors) throws ProvisionException { - this(mavenContext, null, disableP2Mirrors); + this(mavenContext, null, null, disableP2Mirrors); } // constructor for tests public RemoteAgent(MavenContext mavenContext) throws ProvisionException { - this(mavenContext, null, false); + this(mavenContext, null, null, false); } private static IProvisioningAgent createConfiguredProvisioningAgent(MavenContext mavenContext, - boolean disableP2Mirrors, MavenRepositorySettings mavenRepositorySettings) throws ProvisionException { + IProxyService proxyService, boolean disableP2Mirrors, MavenRepositorySettings mavenRepositorySettings) + throws ProvisionException { // TODO set a temporary folder as persistence location AgentBuilder agent = new AgentBuilder(Activator.newProvisioningAgent()); - - // suppress p2.index access - final Transport transport; - if (mavenContext.isOffline()) { - transport = new OfflineTransport(mavenContext); - agent.registerService(Transport.class, transport); + if (!"ecf".equalsIgnoreCase(System.getProperty("tycho.p2.transport"))) { + TychoRepositoryTransport tychoRepositoryTransport = new TychoRepositoryTransport(mavenContext, proxyService, + uri -> { + if (mavenRepositorySettings == null) { + return null; + } + IRepositoryIdManager repositoryIdManager = agent.getService(IRepositoryIdManager.class); + Stream locations = mavenContext.getMavenRepositoryLocations(); + if (repositoryIdManager instanceof RemoteRepositoryLoadingHelper) { + RemoteRepositoryLoadingHelper repositoryLoadingHelper = (RemoteRepositoryLoadingHelper) repositoryIdManager; + locations = Stream.concat(locations, + repositoryLoadingHelper.getKnownMavenRepositoryLocations()); + } + String requestUri = uri.normalize().toASCIIString(); + return locations.sorted((loc1, loc2) -> { + //we wan't the longest prefix match, so first sort all uris by their length ... + String s1 = loc1.getURL().normalize().toASCIIString(); + String s2 = loc2.getURL().normalize().toASCIIString(); + return Long.compare(s2.length(), s1.length()); + }).filter(loc -> { + String prefix = loc.getURL().normalize().toASCIIString(); + return requestUri.startsWith(prefix); + }).map(mavenRepositorySettings::getCredentials).filter(Objects::nonNull).findFirst() + .orElse(null); + }); + agent.getAgent().registerService(CacheManager.SERVICE_NAME, + new TychoRepositoryTransportCacheManager(tychoRepositoryTransport, mavenContext)); + agent.getAgent().registerService(Transport.SERVICE_NAME, tychoRepositoryTransport); } else { - transport = agent.getService(Transport.class); + // suppress p2.index access + final Transport transport; + if (mavenContext.isOffline()) { + transport = new OfflineTransport(mavenContext); + agent.registerService(Transport.class, transport); + } else { + transport = agent.getService(Transport.class); + } + + // cache indices of p2 repositories in the local Maven repository + RemoteRepositoryCacheManager cacheMgr = new RemoteRepositoryCacheManager(transport, mavenContext); + agent.registerService(CacheManager.class, cacheMgr); } - // cache indices of p2 repositories in the local Maven repository - RemoteRepositoryCacheManager cacheMgr = new RemoteRepositoryCacheManager(transport, mavenContext); - agent.registerService(CacheManager.class, cacheMgr); - if (disableP2Mirrors) { addP2MirrorDisablingRepositoryManager(agent, mavenContext.getLogger()); } diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgentManager.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgentManager.java index 289119c6a1..830d8b91f6 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgentManager.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgentManager.java @@ -12,6 +12,7 @@ *******************************************************************************/ package org.eclipse.tycho.p2.remote; +import org.eclipse.core.net.proxy.IProxyService; import org.eclipse.equinox.p2.core.IProvisioningAgent; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings; @@ -33,6 +34,8 @@ public class RemoteAgentManager { // TODO stop when this service is stopped? private IProvisioningAgent cachedAgent; + private IProxyService proxyService; + public RemoteAgentManager(MavenContext mavenContext) { this.mavenContext = mavenContext; } @@ -44,7 +47,7 @@ public RemoteAgentManager() { public synchronized IProvisioningAgent getProvisioningAgent() throws ProvisionException { if (cachedAgent == null) { boolean disableP2Mirrors = getDisableP2MirrorsConfiguration(); - cachedAgent = new RemoteAgent(mavenContext, mavenRepositorySettings, disableP2Mirrors); + cachedAgent = new RemoteAgent(mavenContext, proxyService, mavenRepositorySettings, disableP2Mirrors); } return cachedAgent; } @@ -70,4 +73,8 @@ public void setMavenRepositorySettings(MavenRepositorySettings mavenRepositorySe this.mavenRepositorySettings = mavenRepositorySettings; } + public void setProxyService(IProxyService proxyService) { + this.proxyService = proxyService; + } + } diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteRepositoryLoadingHelper.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteRepositoryLoadingHelper.java index 876ba4045c..3b80aa0fce 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteRepositoryLoadingHelper.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteRepositoryLoadingHelper.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012 SAP AG and others. + * Copyright (c) 2012, 2022 SAP AG and others. * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at @@ -9,13 +9,15 @@ * * Contributors: * SAP AG - initial API and implementation + * Christoph Läubrich - Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.p2.remote; import java.net.URI; import java.net.URISyntaxException; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; import org.eclipse.equinox.p2.core.ProvisionException; import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation; @@ -31,7 +33,7 @@ class RemoteRepositoryLoadingHelper implements IRepositoryIdManager { private final MavenRepositorySettings settings; private final MavenLogger logger; - private Map knownMavenRepositoryIds = new HashMap<>(); + private Map knownMavenRepositoryIds = new ConcurrentHashMap<>(); public RemoteRepositoryLoadingHelper(MavenRepositorySettings settings, MavenLogger logger) { this.settings = settings; @@ -134,4 +136,9 @@ private static boolean certainlyNoRemoteURL(URI location) { return location.isOpaque() || !location.isAbsolute(); } + public Stream getKnownMavenRepositoryLocations() { + return knownMavenRepositoryIds.entrySet().stream() + .map(e -> new MavenRepositoryLocation(e.getValue(), e.getKey())); + } + } diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/SharedHttpCacheStorage.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/SharedHttpCacheStorage.java new file mode 100644 index 0000000000..258ecb0db4 --- /dev/null +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/SharedHttpCacheStorage.java @@ -0,0 +1,592 @@ +/******************************************************************************* + * Copyright (c) 2022 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2.remote; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.Authenticator; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.net.Proxy.Type; +import java.net.SocketAddress; +import java.net.URI; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Base64; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.TimeZone; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.eclipse.core.net.proxy.IProxyData; +import org.eclipse.core.net.proxy.IProxyService; +import org.eclipse.ecf.provider.filetransfer.util.ProxySetupHelper; +import org.eclipse.equinox.internal.p2.repository.AuthenticationFailedException; +import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings.Credentials; +import org.eclipse.tycho.core.shared.MavenLogger; + +public class SharedHttpCacheStorage { + + /** + * Assumes the following minimum caching period for remote files in minutes + */ + //TODO can we sync this with the time where maven updates snapshots? + public static final long MIN_CACHE_PERIOD = Long.getLong("tycho.p2.transport.min-cache-minutes", + TimeUnit.HOURS.toMinutes(1)); + private static final String PROXY_AUTHORIZATION_HEADER = "Proxy-Authorization"; + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String LAST_MODIFIED_HEADER = "Last-Modified"; + private static final String EXPIRES_HEADER = "Expires"; + private static final String CACHE_CONTROL_HEADER = "Cache-Control"; + private static final String MAX_AGE_DIRECTIVE = "max-age"; + private static final String MUST_REVALIDATE_DIRECTIVE = "must-revalidate"; + + private static final String ETAG_HEADER = "ETag"; + + private static final Map storageMap = new HashMap<>(); + + private static final int MAX_IN_MEMORY = 1000; + + private final Map entryCache; + + private CacheConfig cacheConfig; + + private SharedHttpCacheStorage(CacheConfig cacheConfig) { + + this.cacheConfig = cacheConfig; + entryCache = new LinkedHashMap(100, 0.75f, true) { + + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(final Map.Entry eldest) { + return (size() > MAX_IN_MEMORY); + } + + }; + } + + /** + * Fetches the cache entry for this URI + * + * @param uri + * @return + * @throws FileNotFoundException + * if the URI is know to be not found + */ + public CacheEntry getCacheEntry(URI uri, MavenLogger logger) throws FileNotFoundException { + CacheLine cacheLine = getCacheLine(uri); + if (!cacheConfig.update) { //if not updates are forced ... + int code = cacheLine.getResponseCode(); + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + throw new FileNotFoundException(uri.toASCIIString()); + } + if (code == HttpURLConnection.HTTP_MOVED_PERM) { + return getCacheEntry(cacheLine.getRedirect(uri), logger); + } + } + return new CacheEntry() { + + @Override + public long getLastModified(IProxyService proxyService, Function credentialsProvider) + throws IOException { + if (cacheConfig.offline) { + return cacheLine.getLastModified(uri, proxyService, credentialsProvider, + SharedHttpCacheStorage::mavenIsOffline, logger); + } + try { + return cacheLine.fetchLastModified(uri, proxyService, credentialsProvider, logger); + } catch (FileNotFoundException | AuthenticationFailedException e) { + //for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { + if (!cacheConfig.update && cacheLine.getResponseCode() > 0) { + //if we have something cached, use that ... + logger.warn("Request to " + uri + " failed, trying cache instead..."); + return cacheLine.getLastModified(uri, proxyService, credentialsProvider, nil -> e, logger); + } + throw e; + } + } + + @Override + public File getCacheFile(IProxyService proxyService, Function credentialsProvider) + throws IOException { + if (cacheConfig.offline) { + return cacheLine.getFile(uri, proxyService, credentialsProvider, + SharedHttpCacheStorage::mavenIsOffline, logger); + } + try { + return cacheLine.fetchFile(uri, proxyService, credentialsProvider, logger); + } catch (FileNotFoundException | AuthenticationFailedException e) { + //for not found and failed authentication we can't do anything useful + throw e; + } catch (IOException e) { + if (!cacheConfig.update && cacheLine.getResponseCode() > 0) { + //if we have something cached, use that ... + logger.warn("Request to " + uri + " failed, trying cache instead..."); + return cacheLine.getFile(uri, proxyService, credentialsProvider, nil -> e, logger); + } + throw e; + } + } + + }; + } + + private synchronized CacheLine getCacheLine(URI uri) { + File file = new File(cacheConfig.location, + uri.normalize().toASCIIString().replace(':', '/').replaceAll("/+", "/")); + File location; + try { + location = file.getCanonicalFile(); + } catch (IOException e) { + location = file.getAbsoluteFile(); + } + return entryCache.computeIfAbsent(location, CacheLine::new); + + } + + private final class CacheLine { + + private static final String RESPONSE_CODE = "HTTP_RESPONSE_CODE"; + private static final String LAST_UPDATED = "FILE-LAST_UPDATED"; + private static final String STATUS_LINE = "HTTP_STATUS_LINE"; + private final File file; + private final File headerFile; + private Properties header; + private final DateFormat httpDateFormat = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); + + public CacheLine(File file) { + this.file = file; + this.headerFile = new File(file.getParent(), file.getName() + ".headers"); + httpDateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + public synchronized long fetchLastModified(URI uri, IProxyService proxyService, + Function credentialsProvider, MavenLogger logger) throws IOException { + //TODO its very likely that the file is downloaded here if it has changed... so probably just download it right now? + RepositoryAuthenticator authenticator = new RepositoryAuthenticator(getProxyData(proxyService, uri), + credentialsProvider.apply(uri)); + HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(authenticator.getProxy()); + connection.setAuthenticator(authenticator); + connection.setRequestMethod("HEAD"); + authenticator.preemtiveAuth(connection); + connection.connect(); + try { + int code = connection.getResponseCode(); + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + updateHeader(connection, code); + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { + updateHeader(connection, code); + return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(proxyService, + credentialsProvider); + } + return connection.getLastModified(); + } finally { + closeConnection(connection); + } + } + + public synchronized long getLastModified(URI uri, IProxyService proxyService, + Function credentialsProvider, Function notAviableExceptionSupplier, + MavenLogger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { + return SharedHttpCacheStorage.this.getCacheEntry(uri, logger).getLastModified(proxyService, + credentialsProvider); + } + Properties offlineHeader = getHeader(); + Date lastModified = pareHttpDate(offlineHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + if (lastModified != null) { + return lastModified.getTime(); + } + return -1; + } else { + throw notAviableExceptionSupplier.apply(uri); + } + } + + public synchronized File fetchFile(URI uri, IProxyService proxyService, + Function credentialsProvider, MavenLogger logger) throws IOException { + boolean exits = file.isFile(); + if (exits && !mustValidate()) { + return file; + } + RepositoryAuthenticator authenticator = new RepositoryAuthenticator(getProxyData(proxyService, uri), + credentialsProvider.apply(uri)); + HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(authenticator.getProxy()); + connection.setAuthenticator(authenticator); + authenticator.preemtiveAuth(connection); + Properties lastHeader = getHeader(); + if (exits) { + if (lastHeader.containsKey(ETAG_HEADER.toLowerCase())) { + connection.setRequestProperty("If-None-Match", lastHeader.getProperty(ETAG_HEADER.toLowerCase())); + } + if (lastHeader.contains(LAST_MODIFIED_HEADER.toLowerCase())) { + connection.setRequestProperty("If-Modified-Since", + lastHeader.getProperty(LAST_MODIFIED_HEADER.toLowerCase())); + } + } + connection.setInstanceFollowRedirects(false); + connection.connect(); + int code = connection.getResponseCode(); + if (exits && code == HttpURLConnection.HTTP_NOT_MODIFIED) { + updateHeader(connection, getResponseCode()); + return file; + } + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? + } + updateHeader(connection, code); + if (isRedirected(code)) { + closeConnection(connection); + return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger).getCacheFile(proxyService, + credentialsProvider); + } + if (exits) { + FileUtils.forceDelete(file); + } + File tempFile = File.createTempFile("download", ".tmp", file.getParentFile()); + try (InputStream inputStream = connection.getInputStream(); + FileOutputStream os = new FileOutputStream(tempFile)) { + inputStream.transferTo(os); + } catch (IOException e) { + tempFile.delete(); + throw e; + } + FileUtils.moveFile(tempFile, file); + return file; + } + + public synchronized File getFile(URI uri, IProxyService proxyService, + Function credentialsProvider, Function notAviableExceptionSupplier, + MavenLogger logger) throws IOException { + int code = getResponseCode(); + if (code > 0) { + if (isAuthFailure(code)) { + throw new AuthenticationFailedException(); //FIXME why is there no constructor to give a cause? + } + if (isNotFound(code)) { + throw new FileNotFoundException(uri.toString()); + } + if (isRedirected(code)) { + return SharedHttpCacheStorage.this.getCacheEntry(getRedirect(uri), logger) + .getCacheFile(proxyService, credentialsProvider); + } + if (file.isFile()) { + return file; + } + } + throw notAviableExceptionSupplier.apply(uri); + } + + private boolean mustValidate() { + if (cacheConfig.update) { + //user enforced validation + return true; + } + String[] cacheControls = getCacheControl(); + for (String directive : cacheControls) { + if (MUST_REVALIDATE_DIRECTIVE.equals(directive)) { + //server enforced validation + return true; + } + } + Properties properties = getHeader(); + long lastUpdated = parseLong(properties.getProperty(LAST_UPDATED)); + if (lastUpdated + TimeUnit.MINUTES.toMillis(MIN_CACHE_PERIOD) > System.currentTimeMillis()) { + return false; + } + //Cache-Control header with "max-age" directive takes precedence over Expires Header. + for (String directive : cacheControls) { + if (directive.toLowerCase().startsWith(MAX_AGE_DIRECTIVE)) { + long maxAge = parseLong(directive.substring(MAX_AGE_DIRECTIVE.length() + 1)); + if (maxAge <= 0) { + return true; + } + return (lastUpdated + TimeUnit.SECONDS.toMillis(maxAge)) < System.currentTimeMillis(); + } + } + Date expiresDate = pareHttpDate(properties.getProperty(EXPIRES_HEADER.toLowerCase())); + if (expiresDate != null) { + return expiresDate.after(new Date()); + } + return true; + } + + protected long parseLong(String value) { + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + //ignore... + } + } + return 0; + } + + private String[] getCacheControl() { + String property = getHeader().getProperty(CACHE_CONTROL_HEADER); + if (property != null) { + return property.split(",\\s*"); + } + return new String[0]; + } + + protected boolean isAuthFailure(int code) { + return code == HttpURLConnection.HTTP_PROXY_AUTH || code == HttpURLConnection.HTTP_UNAUTHORIZED; + } + + protected void updateHeader(HttpURLConnection connection, int code) throws IOException, FileNotFoundException { + header = new Properties(); + header.setProperty(RESPONSE_CODE, String.valueOf(code)); + header.setProperty(LAST_UPDATED, String.valueOf(System.currentTimeMillis())); + Map> headerFields = connection.getHeaderFields(); + for (var entry : headerFields.entrySet()) { + String key = entry.getKey(); + if (key == null) { + key = STATUS_LINE; + } + key = key.toLowerCase(); + if (AUTHORIZATION_HEADER.equalsIgnoreCase(key) || PROXY_AUTHORIZATION_HEADER.equalsIgnoreCase(key)) { + //Don't store sensitive information here... + continue; + } + if (key.toLowerCase().startsWith("x-")) { + //don't store non default header... + continue; + } + List value = entry.getValue(); + if (value.size() == 1) { + header.put(key, value.get(0)); + } else { + header.put(key, value.stream().collect(Collectors.joining(","))); + } + } + FileUtils.forceMkdir(file.getParentFile()); + try (FileOutputStream out = new FileOutputStream(headerFile)) { + //we store the header here, this might be a 404 response or (permanent) redirect we probably need to work with later on + header.store(out, null); + } + } + + private synchronized Date pareHttpDate(String input) { + if (input != null) { + try { + return httpDateFormat.parse(input); + } catch (ParseException e) { + //can't use it then.. + } + } + return null; + } + + private void closeConnection(HttpURLConnection connection) { + try { + connection.getInputStream().close(); + } catch (IOException e) { + //we just wan't to signal that we are done with this connection... + } + } + + public int getResponseCode() { + return Integer.parseInt(getHeader().getProperty(RESPONSE_CODE, "-1")); + } + + public URI getRedirect(URI base) throws FileNotFoundException { + String location = getHeader().getProperty("location"); + if (location == null) { + throw new FileNotFoundException(base.toASCIIString()); + } + return base.resolve(location); + } + + public Properties getHeader() { + if (header == null) { + header = new Properties(); + if (headerFile.isFile()) { + try { + header.load(new FileInputStream(headerFile)); + } catch (IOException e) { + //can't use the headers then... + } + } + } + return header; + } + } + + private static IProxyData getProxyData(IProxyService proxyService, URI uri) throws IOException { + if (proxyService != null) { + IProxyData[] selected = proxyService.select(uri); + IProxyData proxyData = ProxySetupHelper.selectProxyFromProxies(uri.getScheme(), selected); + if (proxyData != null) { + return proxyData; + } + } + return null; + } + + private static boolean isRedirected(int code) { + return code == HttpURLConnection.HTTP_MOVED_PERM || code == HttpURLConnection.HTTP_MOVED_TEMP; + } + + private static boolean isNotFound(int code) { + return code == HttpURLConnection.HTTP_NOT_FOUND; + } + + public static SharedHttpCacheStorage getStorage(File location, boolean offline, boolean update) { + return storageMap.computeIfAbsent(new CacheConfig(location, offline, update), SharedHttpCacheStorage::new); + } + + private static IOException mavenIsOffline(URI uri) { + return new IOException("maven is currently in offline mode requested URL " + uri + " does not exist locally!"); + } + + private static final class CacheConfig { + public CacheConfig(File location, boolean offline, boolean update) { + this.location = location; + this.offline = offline; + this.update = update; + } + + private final File location; + private final boolean offline; + private final boolean update; + + @Override + public int hashCode() { + return Objects.hash(location, offline, update); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + CacheConfig other = (CacheConfig) obj; + return Objects.equals(location, other.location) && offline == other.offline && update == other.update; + } + } + + private static final class RepositoryAuthenticator extends Authenticator { + + private IProxyData proxyData; + private Credentials credentials; + + public RepositoryAuthenticator(IProxyData proxyData, Credentials credentials) { + this.proxyData = proxyData; + this.credentials = credentials; + } + + public void preemtiveAuth(HttpURLConnection connection) { + // as everything is known and we can't ask the user anyways, preemtive auth is a good choice here to prevent successive requests + addAuthHeader(connection, getPasswordAuthentication(RequestorType.PROXY), PROXY_AUTHORIZATION_HEADER); + addAuthHeader(connection, getPasswordAuthentication(RequestorType.SERVER), AUTHORIZATION_HEADER); + } + + private void addAuthHeader(HttpURLConnection connection, PasswordAuthentication authentication, String header) { + if (authentication == null) { + return; + } + String encoding = Base64.getEncoder().encodeToString( + (authentication.getUserName() + ":" + new String(authentication.getPassword())).getBytes()); + connection.setRequestProperty(header, "Basic " + encoding); + + } + + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return getPasswordAuthentication(getRequestorType()); + } + + protected PasswordAuthentication getPasswordAuthentication(RequestorType type) { + if (type == RequestorType.PROXY) { + if (proxyData != null) { + String userId = proxyData.getUserId(); + if (userId != null) { + String password = proxyData.getPassword(); + return new PasswordAuthentication(userId, + password == null ? new char[0] : password.toCharArray()); + } + } + } else if (type == RequestorType.SERVER) { + if (credentials != null) { + String userName = credentials.getUserName(); + if (userName != null) { + String password = credentials.getPassword(); + return new PasswordAuthentication(userName, + password == null ? new char[0] : password.toCharArray()); + } + } + } + return null; + } + + public Proxy getProxy() { + if (proxyData == null) { + return Proxy.NO_PROXY; + } + return new Proxy(convertType(proxyData), convertAddress(proxyData)); + } + + private static SocketAddress convertAddress(IProxyData data) { + return new InetSocketAddress(data.getHost(), data.getPort()); + } + + private static Type convertType(IProxyData data) { + switch (data.getType()) { + case IProxyData.HTTPS_PROXY_TYPE: + case IProxyData.HTTP_PROXY_TYPE: + return Type.HTTP; + case IProxyData.SOCKS_PROXY_TYPE: + return Type.SOCKS; + default: + return Type.DIRECT; + } + } + } + +} diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransport.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransport.java new file mode 100644 index 0000000000..a76e667881 --- /dev/null +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransport.java @@ -0,0 +1,194 @@ +/******************************************************************************* + * Copyright (c) 2022 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2.remote; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URLConnection; +import java.text.NumberFormat; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.Function; + +import org.apache.commons.io.IOUtils; +import org.eclipse.core.net.proxy.IProxyService; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Status; +import org.eclipse.equinox.internal.p2.repository.AuthenticationFailedException; +import org.eclipse.equinox.internal.provisional.p2.repository.IStateful; +import org.eclipse.equinox.p2.core.IProvisioningAgent; +import org.eclipse.equinox.p2.core.spi.IAgentServiceFactory; +import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings.Credentials; +import org.eclipse.tycho.core.shared.MavenContext; +import org.eclipse.tycho.core.shared.MavenLogger; + +@SuppressWarnings("restriction") +public class TychoRepositoryTransport extends org.eclipse.equinox.internal.p2.repository.Transport + implements IAgentServiceFactory { + + private NumberFormat numberFormat = NumberFormat.getNumberInstance(); + + private MavenContext mavenContext; + private SharedHttpCacheStorage httpCache; + private LongAdder requests = new LongAdder(); + private LongAdder indexRequests = new LongAdder(); + + private IProxyService proxyService; + + private Function credentialsProvider; + + public TychoRepositoryTransport(MavenContext mavenContext, IProxyService proxyService, + Function credentialsProvider) { + this.mavenContext = mavenContext; + this.proxyService = proxyService; + this.credentialsProvider = credentialsProvider; + File cacheLocation = new File(mavenContext.getLocalRepositoryRoot(), ".cache/tycho"); + cacheLocation.mkdirs(); + MavenLogger logger = mavenContext.getLogger(); + logger.info( + "### Using TychoRepositoryTransport for remote P2 access (You can disable this with -Dtycho.p2.transport=ecf) ###"); + logger.info(" Cache location: " + cacheLocation); + logger.info(" Transport mode: " + (mavenContext.isOffline() ? "offline" : "online")); + logger.info(" Update mode: " + (mavenContext.isUpdateSnapshots() ? "forced" : "cache first")); + logger.info(" Minimum cache duration: " + SharedHttpCacheStorage.MIN_CACHE_PERIOD + " minutes"); + logger.info( + " (you can configure this with -Dtycho.p2.transport.min-cache-minutes=)"); + + numberFormat.setMaximumFractionDigits(2); + httpCache = SharedHttpCacheStorage.getStorage(cacheLocation, mavenContext.isOffline(), + mavenContext.isUpdateSnapshots()); + } + + @Override + public IStatus download(URI toDownload, OutputStream target, long startPos, IProgressMonitor monitor) { + if (startPos > 0) { + return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(), + "range downloads are not implemented"); + } + return download(toDownload, target, monitor); + } + + @Override + public IStatus download(URI toDownload, OutputStream target, IProgressMonitor monitor) { + try { + IOUtils.copy(stream(toDownload, monitor), target); + return reportStatus(Status.OK_STATUS, target); + } catch (AuthenticationFailedException e) { + return new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(), + "authentication failed for " + toDownload, e); + } catch (IOException e) { + return reportStatus(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(), + "download from " + toDownload + " failed", e), target); + } catch (CoreException e) { + return reportStatus(e.getStatus(), target); + } + } + + private IStatus reportStatus(IStatus status, OutputStream target) { + if (target instanceof IStateful) { + IStateful stateful = (IStateful) target; + stateful.setStatus(status); + } + return status; + } + + @Override + public synchronized InputStream stream(URI toDownload, IProgressMonitor monitor) + throws FileNotFoundException, CoreException, AuthenticationFailedException { + MavenLogger logger = mavenContext.getLogger(); + if (logger.isExtendedDebugEnabled()) { + logger.debug("Request stream for " + toDownload + "..."); + } + requests.increment(); + if (toDownload.toASCIIString().endsWith("p2.index")) { + indexRequests.increment(); + } + try { + File cachedFile = getCachedFile(toDownload); + if (cachedFile != null) { + if (logger.isExtendedDebugEnabled()) { + logger.debug(" --> routed through http-cache ..."); + } + return new FileInputStream(cachedFile); + } + return toDownload.toURL().openStream(); + } catch (FileNotFoundException e) { + if (logger.isExtendedDebugEnabled()) { + logger.debug(" --> not found!"); + } + throw e; + } catch (IOException e) { + if (logger.isExtendedDebugEnabled()) { + logger.debug(" --> generic error: " + e); + } + throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(), + "download from " + toDownload + " failed", e)); + } finally { + if (logger.isExtendedDebugEnabled()) { + logger.debug("Total number of requests: " + requests.longValue() + " (" + indexRequests.longValue() + + " for p2.index)"); + } + } + } + + @Override + public long getLastModified(URI toDownload, IProgressMonitor monitor) + throws CoreException, FileNotFoundException, AuthenticationFailedException { + //TODO P2 cache manager relies on this method to throw an exception to work correctly + try { + if (isHttp(toDownload)) { + return httpCache.getCacheEntry(toDownload, mavenContext.getLogger()).getLastModified(proxyService, + credentialsProvider); + } + URLConnection connection = toDownload.toURL().openConnection(); + long lastModified = connection.getLastModified(); + connection.getInputStream().close(); + return lastModified; + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new CoreException(new Status(IStatus.ERROR, TychoRepositoryTransport.class.getName(), + "download from " + toDownload + " failed", e)); + } + } + + @Override + public Object createService(IProvisioningAgent agent) { + return this; + } + + public SharedHttpCacheStorage getHttpCache() { + return httpCache; + } + + public File getCachedFile(URI remoteFile) throws IOException { + + if (isHttp(remoteFile)) { + return httpCache.getCacheEntry(remoteFile, mavenContext.getLogger()).getCacheFile(proxyService, + credentialsProvider); + } + return null; + } + + public static boolean isHttp(URI remoteFile) { + String scheme = remoteFile.getScheme(); + return scheme != null && scheme.toLowerCase().startsWith("http"); + } + +} diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransportCacheManager.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransportCacheManager.java new file mode 100644 index 0000000000..8f603fd92c --- /dev/null +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/TychoRepositoryTransportCacheManager.java @@ -0,0 +1,77 @@ +/******************************************************************************* + * Copyright (c) 2022 Christoph Läubrich and others. + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Christoph Läubrich - initial API and implementation + *******************************************************************************/ +package org.eclipse.tycho.p2.remote; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.URIUtil; +import org.eclipse.equinox.internal.p2.repository.CacheManager; +import org.eclipse.equinox.p2.core.ProvisionException; +import org.eclipse.tycho.core.shared.MavenContext; + +@SuppressWarnings("restriction") +public class TychoRepositoryTransportCacheManager extends CacheManager { + + private static final List EXTENSIONS = List.of(".jar", ".xml"); + + private MavenContext mavenContext; + private TychoRepositoryTransport transport; + + public TychoRepositoryTransportCacheManager(TychoRepositoryTransport transport, MavenContext mavenContext) { + super(null, transport); + this.transport = transport; + this.mavenContext = mavenContext; + } + + @Override + public File createCache(URI repositoryLocation, String prefix, IProgressMonitor monitor) + throws IOException, ProvisionException { + if (TychoRepositoryTransport.isHttp(repositoryLocation)) { + for (String extension : EXTENSIONS) { + URI fileLocation = URIUtil.append(repositoryLocation, prefix + extension); + try { + File cachedFile = transport.getCachedFile(fileLocation); + if (cachedFile != null) { + return cachedFile; + } + } catch (FileNotFoundException e) { + continue; + } + } + throw new FileNotFoundException( + "Not found any of " + EXTENSIONS + " for " + repositoryLocation + " with prefix " + prefix); + } + return super.createCache(repositoryLocation, prefix, monitor); + } + + @Override + public File createCacheFromFile(URI remoteFile, IProgressMonitor monitor) throws ProvisionException, IOException { + File cachedFile = transport.getCachedFile(remoteFile); + if (cachedFile != null) { + //no need to cache this twice ... + return cachedFile; + } + return super.createCacheFromFile(remoteFile, monitor); + } + + @Override + protected File getCacheDirectory() { + return new File(mavenContext.getLocalRepositoryRoot(), RemoteRepositoryCacheManager.CACHE_RELPATH); + } + +} diff --git a/tycho-bundles/org.eclipse.tycho.test.utils/META-INF/MANIFEST.MF b/tycho-bundles/org.eclipse.tycho.test.utils/META-INF/MANIFEST.MF index 481b97f756..1c661f2344 100644 --- a/tycho-bundles/org.eclipse.tycho.test.utils/META-INF/MANIFEST.MF +++ b/tycho-bundles/org.eclipse.tycho.test.utils/META-INF/MANIFEST.MF @@ -15,7 +15,8 @@ Require-Bundle: org.junit;bundle-version="[4.8.1,5.0.0)", org.eclipse.jetty.security, org.eclipse.jetty.servlet, org.eclipse.equinox.common;bundle-version="3.6.100", - org.apache.felix.scr;bundle-version="2.1.16" + org.apache.felix.scr;bundle-version="2.1.16", + org.eclipse.core.net Bundle-Vendor: %providerName Export-Package: org.eclipse.tycho.p2.remote.testutil, org.eclipse.tycho.p2.testutil, diff --git a/tycho-bundles/org.eclipse.tycho.test.utils/src/main/java/org/eclipse/tycho/test/util/MavenServiceStubbingTestBase.java b/tycho-bundles/org.eclipse.tycho.test.utils/src/main/java/org/eclipse/tycho/test/util/MavenServiceStubbingTestBase.java index 8d65f78e5a..f9e6206880 100644 --- a/tycho-bundles/org.eclipse.tycho.test.utils/src/main/java/org/eclipse/tycho/test/util/MavenServiceStubbingTestBase.java +++ b/tycho-bundles/org.eclipse.tycho.test.utils/src/main/java/org/eclipse/tycho/test/util/MavenServiceStubbingTestBase.java @@ -12,9 +12,11 @@ *******************************************************************************/ package org.eclipse.tycho.test.util; +import org.eclipse.core.internal.net.ProxyManager; +import org.eclipse.core.net.proxy.IProxyService; import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings; import org.eclipse.tycho.core.shared.MavenContext; -import org.eclipse.tycho.core.shared.MavenContextImpl; +import org.eclipse.tycho.core.shared.MockMavenContext; import org.eclipse.tycho.locking.facade.FileLockService; import org.eclipse.tycho.p2.remote.testutil.MavenRepositorySettingsStub; import org.junit.Before; @@ -43,13 +45,17 @@ public class MavenServiceStubbingTestBase { public StubServiceRegistration fileLockServiceRegistration = new StubServiceRegistration<>( FileLockService.class, new NoopFileLockService()); + @Rule + public StubServiceRegistration proxyServiceRegistration = new StubServiceRegistration<>( + IProxyService.class, ProxyManager.getProxyManager()); + @Before public void initServiceInstances() throws Exception { mavenContextRegistration.registerService(createMavenContext()); } private MavenContext createMavenContext() throws Exception { - MavenContext mavenContext = new MavenContextImpl(temporaryFolder.newFolder("target"), logVerifier.getLogger()) { + MavenContext mavenContext = new MockMavenContext(temporaryFolder.newFolder("target"), logVerifier.getLogger()) { @Override public String getExtension(String artifactType) { diff --git a/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/MavenContextConfigurator.java b/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/MavenContextConfigurator.java index d38029966f..ec8873ad9a 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/MavenContextConfigurator.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/MavenContextConfigurator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2011, 2020 SAP AG and others. + * Copyright (c) 2011, 2022 SAP AG and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -8,12 +8,21 @@ * Contributors: * SAP AG - initial API and implementation * Christoph Läubrich - Bug 564363 - Make ReactorProject available in MavenContext + * Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.osgi.configuration; import java.io.File; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Collection; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.maven.artifact.handler.ArtifactHandler; import org.apache.maven.artifact.handler.manager.ArtifactHandlerManager; @@ -28,6 +37,8 @@ import org.eclipse.sisu.equinox.embedder.EmbeddedEquinox; import org.eclipse.sisu.equinox.embedder.EquinoxLifecycleListener; import org.eclipse.tycho.core.osgitools.DefaultReactorProject; +import org.eclipse.tycho.core.p2.P2ArtifactRepositoryLayout; +import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation; import org.eclipse.tycho.core.shared.MavenContext; import org.eclipse.tycho.core.shared.MavenContextImpl; import org.eclipse.tycho.osgi.adapters.MavenLoggerAdapter; @@ -50,6 +61,15 @@ public void afterFrameworkStarted(EmbeddedEquinox framework) { File localRepoRoot = new File(session.getLocalRepository().getBasedir()); MavenLoggerAdapter mavenLogger = new MavenLoggerAdapter(logger, false); Properties globalProps = getGlobalProperties(session); + List repositoryLocations = session.getProjects().stream() + .map(MavenProject::getRemoteArtifactRepositories).flatMap(Collection::stream) + .filter(r -> r.getLayout() instanceof P2ArtifactRepositoryLayout).map(r -> { + try { + return new MavenRepositoryLocation(r.getId(), new URL(r.getUrl()).toURI()); + } catch (MalformedURLException | URISyntaxException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toUnmodifiableList()); MavenContextImpl mavenContext = new MavenContextImpl(localRepoRoot, session.isOffline(), mavenLogger, globalProps) { @@ -62,6 +82,17 @@ public String getExtension(String artifactType) { return handler.getExtension(); } + @Override + public boolean isUpdateSnapshots() { + return session.getRequest().isUpdateSnapshots(); + } + + @Override + public Stream getMavenRepositoryLocations() { + + return repositoryLocations.stream(); + } + }; for (MavenProject project : session.getProjects()) { mavenContext.addProject(DefaultReactorProject.adapt(project)); diff --git a/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/RepositorySettingsConfigurator.java b/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/RepositorySettingsConfigurator.java index 73de26bd17..b4cf55cd2b 100644 --- a/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/RepositorySettingsConfigurator.java +++ b/tycho-core/src/main/java/org/eclipse/tycho/osgi/configuration/RepositorySettingsConfigurator.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012 SAP AG and others. + * Copyright (c) 2012, 2022 SAP AG and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -7,6 +7,7 @@ * * Contributors: * SAP AG - initial API and implementation + * Christoph Läubrich - Issue #797 - Implement a caching P2 transport *******************************************************************************/ package org.eclipse.tycho.osgi.configuration; @@ -88,7 +89,7 @@ public MavenRepositorySettings.Credentials getCredentials(MavenRepositoryLocatio SettingsDecryptionResult result = decrypter.decryptAndLogProblems(serverSettings); Server decryptedServer = result.getServer(); return new MavenRepositorySettings.Credentials(decryptedServer.getUsername(), - decryptedServer.getPassword()); + decryptedServer.getPassword(), location.getURL()); } return null; }