Issue eclipse-tycho#797 - add a custom tycho transport

RELEASE_NOTES.md

@@ -4,6 +4,31 @@ This page describes the noteworthy improvements provided by each release of Ecli
 
 ## 3.0.0 (under development)
 
+### Improved P2 transport for more efficiently http-cache handling and improved offline mode
+
+P2 default transport is more designed as a weak cache that assumes the user is always online.
+While for an IDE that might be sufficient as target resolution is only performed once in a while and updates are triggered by explicit user request, for tycho this does not work well:
+
+- Builds are often trigger on each code change, requiring repeated target resolution
+- Builds might be asked to run in an offline mode
+- If there is a temporary server outrage one might to fallback to the previous state for this build instead of fail completely
+- Build times are often a rare resource one don't want to waste waiting for servers, bandwidth might even be limited or you have to pay for it
+
+Because of this, Tycho now includes a brand new caching P2 transport that allows advanced aching, offline handling and fallback to cache in case of server failures. The transport is enabled by default so nothing has to be done, just in case you want the old behavior you can set `-D=tycho.p2.transport=ecf` beside that the following properties might be interesting:
+
+#### Force cache-revalidation
+If you run maven with the `-U` switch Tycho revalidates the cache, this is useful if you have changed an updatesite in an unusual way (e.g. adding new index files) as tycho now also caches not found items to speed-up certain scenarios where many non existing files are queried.
+
+#### Configure minimum caching age
+
+Some servers don't provide you with sufficient caching information, for this purpose, tychy by default assumes a minimum caching age of one hour. You can switch this off, or configure a longer delay by using `-Dtycho.p2.transport.min-cache-minutes=<desired minimum in minutes>`.
+Choosing a sensible value could greatly improve your build times and lower banYdwith usage. If your build contains a mixture of released and 'snapshot' sites you have the following options:
+
+1. Consider adding a mirror to your settings.xml for the snapshot page that point to a file-local copy (e.g. output of another build)
+2. Configure the webserver of your snapshot site with the `Cache-Control: must-revalidate` header in which case tycho ignores any minimum age
+3. Use `-Dtycho.p2.transport.min-cache-minutes=0` this will still improve the time to resolve the target
+
+
 ### Automatic generation of PDE source bundles for pom-first bundles
 
 PDE requires some special headers to detect a bundle as a "Source Bundle", there is now a new mojo `tycho-source-plugin:generate-pde-source-header` that support this, it requires the following configuration:

tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositoryLocation.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2012 SAP AG and others.
+ * Copyright (c) 2012, 2022 SAP AG and others.
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -9,6 +9,7 @@
  *
  * Contributors:
  *    SAP AG - initial API and implementation
+ *    Christoph Läubrich - Issue #797 - Implement a caching P2 transport  
  *******************************************************************************/
 package org.eclipse.tycho.core.resolver.shared;
 

tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/resolver/shared/MavenRepositorySettings.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2012, 2013 SAP AG and others.
+ * Copyright (c) 2012, 2022 SAP AG and others.
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -9,9 +9,12 @@
  *
  * Contributors:
  *    SAP AG - initial API and implementation
+ *    Christoph Läubrich - Issue #797 - Implement a caching P2 transport  
  *******************************************************************************/
 package org.eclipse.tycho.core.resolver.shared;
 
+import java.net.URI;
+
 /**
  * Provides the mirror configuration and credentials from the Maven settings for loading remote p2
  * repositories.
@@ -21,10 +24,12 @@ public interface MavenRepositorySettings {
     public final class Credentials {
         private final String userName;
         private final String password;
+        private final URI url;
 
-        public Credentials(String userName, String password) {
+        public Credentials(String userName, String password, URI uri) {
             this.userName = userName;
             this.password = password;
+            this.url = uri;
         }
 
         public String getUserName() {
@@ -34,6 +39,10 @@ public String getUserName() {
         public String getPassword() {
             return password;
         }
+
+        public URI getURI() {
+            return url;
+        }
     }
 
     /**

tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MavenContext.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2011, 2020 SAP AG and others.
+ * Copyright (c) 2011, 2022 SAP AG and others.
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -16,8 +16,10 @@
 import java.io.File;
 import java.util.Collection;
 import java.util.Properties;
+import java.util.stream.Stream;
 
 import org.eclipse.tycho.ReactorProject;
+import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation;
 
 /**
  * Makes maven information which is constant for the whole maven session available as a service to
@@ -34,6 +36,11 @@ public interface MavenContext {
      */
     public boolean isOffline();
 
+    /**
+     * whether maven was started with the update-snapshots mode (CLI option "-U")
+     */
+    boolean isUpdateSnapshots();
+
     /**
      * Session-global properties merged from (in order of precedence)
      * <ol>
@@ -59,4 +66,10 @@ public interface MavenContext {
      */
     public String getExtension(String artifactType);
 
+    /**
+     * 
+     * @return a collection of all {@link MavenRepositoryLocation}s know to the maven context
+     */
+    Stream<MavenRepositoryLocation> getMavenRepositoryLocations();
+
 }

tycho-bundles/org.eclipse.tycho.core.shared/src/main/java/org/eclipse/tycho/core/shared/MockMavenContext.java

@@ -14,9 +14,11 @@
 
 import java.io.File;
 import java.util.Properties;
+import java.util.stream.Stream;
 
 import org.eclipse.tycho.ArtifactType;
 import org.eclipse.tycho.PackagingType;
+import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation;
 
 public class MockMavenContext extends MavenContextImpl {
 
@@ -59,4 +61,14 @@ public String getExtension(String artifactType) {
         }
     }
 
+    @Override
+    public boolean isUpdateSnapshots() {
+        return false;
+    }
+
+    @Override
+    public Stream<MavenRepositoryLocation> getMavenRepositoryLocations() {
+        return Stream.empty();
+    }
+
 }

tycho-bundles/org.eclipse.tycho.p2.maven.repository.tests/src/test/java/org/eclipse/tycho/repository/local/GAVArtifactDescriptorTest.java

@@ -21,7 +21,7 @@
 import org.eclipse.equinox.p2.metadata.IArtifactKey;
 import org.eclipse.equinox.p2.metadata.Version;
 import org.eclipse.equinox.p2.repository.artifact.spi.ArtifactDescriptor;
-import org.eclipse.tycho.core.shared.MavenContextImpl;
+import org.eclipse.tycho.core.shared.MockMavenContext;
 import org.eclipse.tycho.p2.repository.GAV;
 import org.eclipse.tycho.p2.repository.MavenRepositoryCoordinates;
 import org.junit.Test;
@@ -170,7 +170,7 @@ public void testGetLocalRepositoryPath() {
                 OTHER_EXTENSION);
         subject = new GAVArtifactDescriptor(createP2Descriptor(), coordinates);
 
-        assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MavenContextImpl(null, false, null, null) {
+        assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MockMavenContext(null, false, null, null) {
 
             @Override
             public String getExtension(String artifactType) {
@@ -185,7 +185,7 @@ public void testGetLocalRepositoryPathWithDefaults() {
                 DEFAULT_EXTENSION);
         subject = new GAVArtifactDescriptor(createP2Descriptor(), coordinates);
 
-        assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MavenContextImpl(null, false, null, null) {
+        assertThat(subject.getMavenCoordinates().getLocalRepositoryPath(new MockMavenContext(null, false, null, null) {
 
             @Override
             public String getExtension(String artifactType) {

tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/manager/ReactorRepositoryManagerTest.java

@@ -77,7 +77,7 @@ public void testReactorRepositoryManagerFacadeServiceAvailability() throws Excep
     @Test
     public void testTargetPlatformComputationInIntegration() throws Exception {
         subject = getService(ReactorRepositoryManagerFacade.class);
-
+        assertThat(subject, is(notNullValue()));
         ReactorProject currentProject = new ReactorProjectStub("reactor-artifact");
 
         TargetPlatformConfigurationStub tpConfig = new TargetPlatformConfigurationStub();

tycho-bundles/org.eclipse.tycho.p2.resolver.impl.test/src/test/java/org/eclipse/tycho/p2/remote/RemoteAgentMavenMirrorsTest.java

@@ -54,10 +54,15 @@ public class RemoteAgentMavenMirrorsTest {
     public void initSubject() throws Exception {
         File localRepository = tempManager.newFolder("localRepo");
         MavenContext mavenContext = new MockMavenContext(localRepository, OFFLINE, logVerifier.getLogger(),
-                new Properties());
+                new Properties()) {
+            @Override
+            public boolean isUpdateSnapshots() {
+                return true;
+            }
+        };
 
         mavenRepositorySettings = new MavenRepositorySettingsStub();
-        subject = new RemoteAgent(mavenContext, mavenRepositorySettings, OFFLINE);
+        subject = new RemoteAgent(mavenContext, null, mavenRepositorySettings, OFFLINE);
     }
 
     @Test

tycho-bundles/org.eclipse.tycho.p2.resolver.impl/META-INF/MANIFEST.MF

@@ -33,6 +33,7 @@ Export-Package: org.eclipse.tycho.p2.impl;x-friends:="org.eclipse.tycho.p2.impl.
  org.eclipse.tycho.p2.target.ee;x-friends:="org.eclipse.tycho.p2.tools.impl",
  org.eclipse.tycho.p2.util.resolution
 Import-Package: org.apache.commons.io;version="2.8.0",
+ org.eclipse.ecf.provider.filetransfer.util;version="3.2.0",
  org.eclipse.tycho,
  org.eclipse.tycho.artifacts,
  org.eclipse.tycho.core.ee.shared,

tycho-bundles/org.eclipse.tycho.p2.resolver.impl/OSGI-INF/remoteAgentManager.xml

@@ -6,4 +6,5 @@
    </service>
    <reference bind="setMavenContext" cardinality="1..1" interface="org.eclipse.tycho.core.shared.MavenContext" name="MavenContext" policy="static"/>
    <reference bind="setMavenRepositorySettings" cardinality="1..1" interface="org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings" name="MavenRepositorySettings" policy="static"/>
+   <reference bind="setProxyService" cardinality="1..1" interface="org.eclipse.core.net.proxy.IProxyService" name="IProxyService" policy="static"/>
 </scr:component>

tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/CacheEntry.java

@@ -0,0 +1,28 @@
+/*******************************************************************************
+ * Copyright (c) 2022 Christoph Läubrich and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Christoph Läubrich - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.tycho.p2.remote;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.function.Function;
+
+import org.eclipse.core.net.proxy.IProxyService;
+import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings.Credentials;
+
+public interface CacheEntry {
+
+    long getLastModified(IProxyService proxyService, Function<URI, Credentials> credentialsProvider) throws IOException;
+
+    File getCacheFile(IProxyService proxyService, Function<URI, Credentials> credentialsProvider) throws IOException;
+}

tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/remote/RemoteAgent.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2012 SAP AG and others.
+ * Copyright (c) 2012, 2022 SAP AG and others.
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
  * which accompanies this distribution, and is available at
@@ -9,15 +9,21 @@
  *
  * Contributors:
  *    SAP AG - initial API and implementation
+ *    Christoph Läubrich - Issue #797 - Implement a caching P2 transport  
  *******************************************************************************/
 package org.eclipse.tycho.p2.remote;
 
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import org.eclipse.core.net.proxy.IProxyService;
 import org.eclipse.equinox.internal.p2.repository.CacheManager;
 import org.eclipse.equinox.internal.p2.repository.Transport;
 import org.eclipse.equinox.p2.core.IProvisioningAgent;
 import org.eclipse.equinox.p2.core.ProvisionException;
 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager;
 import org.eclipse.equinox.p2.repository.metadata.IMetadataRepositoryManager;
+import org.eclipse.tycho.core.resolver.shared.MavenRepositoryLocation;
 import org.eclipse.tycho.core.resolver.shared.MavenRepositorySettings;
 import org.eclipse.tycho.core.shared.MavenContext;
 import org.eclipse.tycho.core.shared.MavenLogger;
@@ -28,39 +34,70 @@ public class RemoteAgent implements IProvisioningAgent {
 
     private IProvisioningAgent delegate;
 
-    public RemoteAgent(MavenContext mavenContext, MavenRepositorySettings mavenRepositorySettings,
-            boolean disableMirrors) throws ProvisionException {
-        this.delegate = createConfiguredProvisioningAgent(mavenContext, disableMirrors, mavenRepositorySettings);
+    public RemoteAgent(MavenContext mavenContext, IProxyService proxyService,
+            MavenRepositorySettings mavenRepositorySettings, boolean disableMirrors) throws ProvisionException {
+        this.delegate = createConfiguredProvisioningAgent(mavenContext, proxyService, disableMirrors,
+                mavenRepositorySettings);
     }
 
     // constructor for tests
     RemoteAgent(MavenContext mavenContext, boolean disableP2Mirrors) throws ProvisionException {
-        this(mavenContext, null, disableP2Mirrors);
+        this(mavenContext, null, null, disableP2Mirrors);
     }
 
     // constructor for tests
     public RemoteAgent(MavenContext mavenContext) throws ProvisionException {
-        this(mavenContext, null, false);
+        this(mavenContext, null, null, false);
     }
 
     private static IProvisioningAgent createConfiguredProvisioningAgent(MavenContext mavenContext,
-            boolean disableP2Mirrors, MavenRepositorySettings mavenRepositorySettings) throws ProvisionException {
+            IProxyService proxyService, boolean disableP2Mirrors, MavenRepositorySettings mavenRepositorySettings)
+            throws ProvisionException {
         // TODO set a temporary folder as persistence location
         AgentBuilder agent = new AgentBuilder(Activator.newProvisioningAgent());
-
-        // suppress p2.index access
-        final Transport transport;
-        if (mavenContext.isOffline()) {
-            transport = new OfflineTransport(mavenContext);
-            agent.registerService(Transport.class, transport);
+        if (!"ecf".equalsIgnoreCase(System.getProperty("tycho.p2.transport"))) {
+            TychoRepositoryTransport tychoRepositoryTransport = new TychoRepositoryTransport(mavenContext, proxyService,
+                    uri -> {
+                        if (mavenRepositorySettings == null) {
+                            return null;
+                        }
+                        IRepositoryIdManager repositoryIdManager = agent.getService(IRepositoryIdManager.class);
+                        Stream<MavenRepositoryLocation> locations = mavenContext.getMavenRepositoryLocations();
+                        if (repositoryIdManager instanceof RemoteRepositoryLoadingHelper) {
+                            RemoteRepositoryLoadingHelper repositoryLoadingHelper = (RemoteRepositoryLoadingHelper) repositoryIdManager;
+                            locations = Stream.concat(locations,
+                                    repositoryLoadingHelper.getKnownMavenRepositoryLocations());
+                        }
+                        String requestUri = uri.normalize().toASCIIString();
+                        return locations.sorted((loc1, loc2) -> {
+                            //we wan't the longest prefix match, so first sort all uris by their length ...
+                            String s1 = loc1.getURL().normalize().toASCIIString();
+                            String s2 = loc2.getURL().normalize().toASCIIString();
+                            return Long.compare(s2.length(), s1.length());
+                        }).filter(loc -> {
+                            String prefix = loc.getURL().normalize().toASCIIString();
+                            return requestUri.startsWith(prefix);
+                        }).map(mavenRepositorySettings::getCredentials).filter(Objects::nonNull).findFirst()
+                                .orElse(null);
+                    });
+            agent.getAgent().registerService(CacheManager.SERVICE_NAME,
+                    new TychoRepositoryTransportCacheManager(tychoRepositoryTransport, mavenContext));
+            agent.getAgent().registerService(Transport.SERVICE_NAME, tychoRepositoryTransport);
         } else {
-            transport = agent.getService(Transport.class);
+            // suppress p2.index access
+            final Transport transport;
+            if (mavenContext.isOffline()) {
+                transport = new OfflineTransport(mavenContext);
+                agent.registerService(Transport.class, transport);
+            } else {
+                transport = agent.getService(Transport.class);
+            }
+
+            // cache indices of p2 repositories in the local Maven repository
+            RemoteRepositoryCacheManager cacheMgr = new RemoteRepositoryCacheManager(transport, mavenContext);
+            agent.registerService(CacheManager.class, cacheMgr);
         }
 
-        // cache indices of p2 repositories in the local Maven repository
-        RemoteRepositoryCacheManager cacheMgr = new RemoteRepositoryCacheManager(transport, mavenContext);
-        agent.registerService(CacheManager.class, cacheMgr);
-
         if (disableP2Mirrors) {
             addP2MirrorDisablingRepositoryManager(agent, mavenContext.getLogger());
         }