Factories for private Gitlab repository with pre-created personal access token

wsmaster/che-core-api-factory-bitbucket-server/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketServerAuthorizingFileContentProvider.java

@@ -11,101 +11,23 @@
  */
 package org.eclipse.che.api.factory.server.bitbucket;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Optional;
+import org.eclipse.che.api.factory.server.scm.AuthorizingFileContentProvider;
 import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
-import org.eclipse.che.api.factory.server.scm.PersonalAccessToken;
 import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
-import org.eclipse.che.api.factory.server.scm.exception.ScmCommunicationException;
-import org.eclipse.che.api.factory.server.scm.exception.ScmConfigurationPersistenceException;
-import org.eclipse.che.api.factory.server.scm.exception.ScmUnauthorizedException;
-import org.eclipse.che.api.factory.server.scm.exception.UnknownScmProviderException;
-import org.eclipse.che.api.factory.server.scm.exception.UnsatisfiedScmPreconditionException;
-import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
 import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
-import org.eclipse.che.api.workspace.server.devfile.exception.DevfileException;
-import org.eclipse.che.commons.env.EnvironmentContext;
 
 /**
  * Bitbucket specific file content provider. Files are retrieved using bitbucket REST API and
  * personal access token based authentication is performed during requests.
  */
-public class BitbucketServerAuthorizingFileContentProvider implements FileContentProvider {
-
-  private final URLFetcher urlFetcher;
-  private final BitbucketUrl bitbucketUrl;
-  private final GitCredentialManager gitCredentialManager;
-  private final PersonalAccessTokenManager personalAccessTokenManager;
+public class BitbucketServerAuthorizingFileContentProvider
+    extends AuthorizingFileContentProvider<BitbucketUrl> {
 
   public BitbucketServerAuthorizingFileContentProvider(
       BitbucketUrl bitbucketUrl,
       URLFetcher urlFetcher,
       GitCredentialManager gitCredentialManager,
       PersonalAccessTokenManager personalAccessTokenManager) {
-    this.bitbucketUrl = bitbucketUrl;
-    this.urlFetcher = urlFetcher;
-    this.gitCredentialManager = gitCredentialManager;
-    this.personalAccessTokenManager = personalAccessTokenManager;
-  }
-
-  @Override
-  public String fetchContent(String fileURL) throws IOException, DevfileException {
-    String requestURL;
-    try {
-      if (new URI(fileURL).isAbsolute()) {
-        requestURL = fileURL;
-      } else {
-        // since files retrieved via REST, we cannot use path symbols like . ./ so cut them off
-        requestURL = bitbucketUrl.rawFileLocation(fileURL.replaceAll("^[/.]+", ""));
-      }
-    } catch (URISyntaxException e) {
-      throw new DevfileException(e.getMessage(), e);
-    }
-    try {
-      Optional<PersonalAccessToken> token =
-          personalAccessTokenManager.get(
-              EnvironmentContext.getCurrent().getSubject(), bitbucketUrl.getHostName());
-      if (token.isPresent()) {
-        PersonalAccessToken personalAccessToken = token.get();
-        String content = urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
-        gitCredentialManager.createOrReplace(personalAccessToken);
-        return content;
-      } else {
-        try {
-          return urlFetcher.fetch(requestURL);
-        } catch (IOException exception) {
-          // unable to determine exact cause, so let's just try to authorize...
-          try {
-            PersonalAccessToken personalAccessToken =
-                personalAccessTokenManager.fetchAndSave(
-                    EnvironmentContext.getCurrent().getSubject(), bitbucketUrl.getHostName());
-            String content =
-                urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
-            gitCredentialManager.createOrReplace(personalAccessToken);
-            return content;
-          } catch (ScmUnauthorizedException
-              | ScmCommunicationException
-              | UnknownScmProviderException e) {
-            throw new DevfileException(e.getMessage(), e);
-          }
-        }
-      }
-
-    } catch (IOException e) {
-      throw new IOException(
-          String.format(
-              "Failed to fetch a content from URL %s. Make sure the URL"
-                  + " is correct. Additionally, if you're using "
-                  + " relative form, make sure the referenced files are actually stored"
-                  + " relative to the devfile on the same host,"
-                  + " or try to specify URL in absolute form. The current attempt to download"
-                  + " the file failed with the following error message: %s",
-              fileURL, e.getMessage()),
-          e);
-    } catch (ScmConfigurationPersistenceException | UnsatisfiedScmPreconditionException e) {
-      throw new DevfileException(e.getMessage(), e);
-    }
+    super(bitbucketUrl, urlFetcher, personalAccessTokenManager, gitCredentialManager);
   }
 }

wsmaster/che-core-api-factory-github/src/main/java/org/eclipse/che/api/factory/server/github/GithubUrl.java

@@ -31,6 +31,8 @@
  */
 public class GithubUrl implements RemoteFactoryUrl {
 
+  private static final String HOSTNAME = "https://github.com/";
+
   /** Username part of github URL */
   private String username;
 
@@ -160,12 +162,17 @@ public String rawFileLocation(String fileName) {
         .toString();
   }
 
+  @Override
+  public String getHostName() {
+    return HOSTNAME;
+  }
+
   /**
    * Provides location to the repository part of the full github URL.
    *
    * @return location of the repository.
    */
   protected String repositoryLocation() {
-    return "https://github.com/" + this.username + "/" + this.repository;
+    return HOSTNAME + this.username + "/" + this.repository;
   }
 }

wsmaster/che-core-api-factory-gitlab/src/main/java/org/eclipse/che/api/factory/server/gitlab/GitlabAuthorizingFileContentProvider.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.api.factory.server.gitlab;
+
+import org.eclipse.che.api.factory.server.scm.AuthorizingFileContentProvider;
+import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
+import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
+import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
+
+/** Gitlab specific authorizing file content provider. */
+class GitlabAuthorizingFileContentProvider extends AuthorizingFileContentProvider<GitlabUrl> {
+
+  GitlabAuthorizingFileContentProvider(
+      GitlabUrl githubUrl,
+      URLFetcher urlFetcher,
+      GitCredentialManager gitCredentialManager,
+      PersonalAccessTokenManager personalAccessTokenManager) {
+    super(githubUrl, urlFetcher, personalAccessTokenManager, gitCredentialManager);
+  }
+}

wsmaster/che-core-api-factory-gitlab/src/main/java/org/eclipse/che/api/factory/server/gitlab/GitlabFactoryParametersResolver.java

@@ -22,6 +22,8 @@
 import org.eclipse.che.api.core.ApiException;
 import org.eclipse.che.api.core.BadRequestException;
 import org.eclipse.che.api.factory.server.DefaultFactoryParameterResolver;
+import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
+import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
 import org.eclipse.che.api.factory.server.urlfactory.URLFactoryBuilder;
 import org.eclipse.che.api.factory.shared.dto.FactoryDto;
 import org.eclipse.che.api.factory.shared.dto.FactoryMetaDto;
@@ -40,11 +42,20 @@ public class GitlabFactoryParametersResolver extends DefaultFactoryParameterReso
 
   private final GitlabUrlParser gitlabURLParser;
 
+  private final GitCredentialManager gitCredentialManager;
+  private final PersonalAccessTokenManager personalAccessTokenManager;
+
   @Inject
   public GitlabFactoryParametersResolver(
-      URLFactoryBuilder urlFactoryBuilder, URLFetcher urlFetcher, GitlabUrlParser gitlabURLParser) {
+      URLFactoryBuilder urlFactoryBuilder,
+      URLFetcher urlFetcher,
+      GitlabUrlParser gitlabURLParser,
+      GitCredentialManager gitCredentialManager,
+      PersonalAccessTokenManager personalAccessTokenManager) {
     super(urlFactoryBuilder, urlFetcher);
     this.gitlabURLParser = gitlabURLParser;
+    this.gitCredentialManager = gitCredentialManager;
+    this.personalAccessTokenManager = personalAccessTokenManager;
   }
 
   /**
@@ -76,7 +87,8 @@ public FactoryMetaDto createFactory(@NotNull final Map<String, String> factoryPa
     return urlFactoryBuilder
         .createFactoryFromDevfile(
             gitlabUrl,
-            new GitlabFileContentProvider(gitlabUrl, urlFetcher),
+            new GitlabAuthorizingFileContentProvider(
+                gitlabUrl, urlFetcher, gitCredentialManager, personalAccessTokenManager),
             extractOverrideParams(factoryParameters))
         .orElseGet(() -> newDto(FactoryDto.class).withV(CURRENT_VERSION).withSource("repo"))
         .acceptVisitor(new GitlabFactoryVisitor(gitlabUrl));

wsmaster/che-core-api-factory-gitlab/src/main/java/org/eclipse/che/api/factory/server/gitlab/GitlabUrl.java

@@ -11,8 +11,9 @@
  */
 package org.eclipse.che.api.factory.server.gitlab;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
+import static java.net.URLEncoder.encode;
 
+import com.google.common.base.Charsets;
 import com.google.common.base.Strings;
 import java.util.ArrayList;
 import java.util.List;
@@ -185,15 +186,23 @@ public String location() {
    * @return location of specified file in a repository
    */
   public String rawFileLocation(String fileName) {
-    StringJoiner joiner = new StringJoiner("/").add(hostName).add(username).add(project);
-    if (repository != null) {
-      joiner.add(repository);
-    }
-    joiner.add("-").add("raw").add(firstNonNull(branch, "master"));
-    if (subfolder != null) {
-      joiner.add(subfolder);
+    String resultUrl =
+        new StringJoiner("/")
+            .add(hostName)
+            .add("api/v4/projects")
+            // use URL-encoded path to the project as a selector instead of id
+            .add(encode(username + "/" + project, Charsets.UTF_8))
+            .add("repository")
+            .add("files")
+            .add(fileName)
+            .add("raw")
+            .toString();
+    if (branch != null) {
+      resultUrl = resultUrl + "?ref=" + branch;
+    } else {
+      resultUrl = resultUrl + "?ref=master";
     }
-    return joiner.add(fileName).toString();
+    return resultUrl;
   }
 
   /**

wsmaster/che-core-api-factory-gitlab/src/test/java/org/eclipse/che/api/factory/server/gitlab/GitlabFileContentProviderTest.java → wsmaster/che-core-api-factory-gitlab/src/test/java/org/eclipse/che/api/factory/server/gitlab/GitlabAuthorizingFileContentProviderTest.java

@@ -14,12 +14,21 @@
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.verify;
 
+import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
+import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
 import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
 import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
+import org.mockito.Mock;
 import org.mockito.Mockito;
+import org.mockito.testng.MockitoTestNGListener;
+import org.testng.annotations.Listeners;
 import org.testng.annotations.Test;
 
-public class GitlabFileContentProviderTest {
+@Listeners(MockitoTestNGListener.class)
+public class GitlabAuthorizingFileContentProviderTest {
+
+  @Mock private GitCredentialManager gitCredentialsManager;
+  @Mock private PersonalAccessTokenManager personalAccessTokenManager;
 
   @Test
   public void shouldExpandRelativePaths() throws Exception {
@@ -29,18 +38,26 @@ public void shouldExpandRelativePaths() throws Exception {
             .withHostName("https://gitlab.net")
             .withUsername("eclipse")
             .withProject("che");
-    FileContentProvider fileContentProvider = new GitlabFileContentProvider(gitlabUrl, urlFetcher);
+    FileContentProvider fileContentProvider =
+        new GitlabAuthorizingFileContentProvider(
+            gitlabUrl, urlFetcher, gitCredentialsManager, personalAccessTokenManager);
     fileContentProvider.fetchContent("devfile.yaml");
-    verify(urlFetcher).fetch(eq("https://gitlab.net/eclipse/che/-/raw/master/devfile.yaml"));
+    verify(urlFetcher)
+        .fetch(
+            eq(
+                "https://gitlab.net/api/v4/projects/eclipse%2Fche/repository/files/devfile.yaml/raw?ref=master"));
   }
 
   @Test
   public void shouldPreserveAbsolutePaths() throws Exception {
     URLFetcher urlFetcher = Mockito.mock(URLFetcher.class);
     GitlabUrl gitlabUrl =
         new GitlabUrl().withHostName("gitlab.net").withUsername("eclipse").withProject("che");
-    FileContentProvider fileContentProvider = new GitlabFileContentProvider(gitlabUrl, urlFetcher);
-    String url = "https://gitlab.net/eclipse/che/-/raw/master/devfile.yaml";
+    FileContentProvider fileContentProvider =
+        new GitlabAuthorizingFileContentProvider(
+            gitlabUrl, urlFetcher, gitCredentialsManager, personalAccessTokenManager);
+    String url =
+        "https://gitlab.net/api/v4/projects/eclipse%2Fche/repository/files/devfile.yaml/raw?ref=master";
     fileContentProvider.fetchContent(url);
     verify(urlFetcher).fetch(eq(url));
   }

wsmaster/che-core-api-factory-gitlab/src/test/java/org/eclipse/che/api/factory/server/gitlab/GitlabUrlTest.java

@@ -11,6 +11,7 @@
  */
 package org.eclipse.che.api.factory.server.gitlab;
 
+import static java.lang.String.format;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.when;
 import static org.testng.Assert.assertEquals;
@@ -57,30 +58,32 @@ public void checkDevfileLocation(String repoUrl, String fileUrl) {
     GitlabUrl gitlabUrl = gitlabUrlParser.parse(repoUrl);
     assertEquals(gitlabUrl.devfileFileLocations().size(), 2);
     Iterator<DevfileLocation> iterator = gitlabUrl.devfileFileLocations().iterator();
-    assertEquals(iterator.next().location(), fileUrl + "devfile.yaml");
-
-    assertEquals(iterator.next().location(), fileUrl + "foo.bar");
+    assertEquals(iterator.next().location(), format(fileUrl, "devfile.yaml"));
+    assertEquals(iterator.next().location(), format(fileUrl, "foo.bar"));
   }
 
   @DataProvider
   public static Object[][] urlsProvider() {
     return new Object[][] {
-      {"https://gitlab.net/eclipse/che.git", "https://gitlab.net/eclipse/che/-/raw/master/"},
+      {
+        "https://gitlab.net/eclipse/che.git",
+        "https://gitlab.net/api/v4/projects/eclipse%%2Fche/repository/files/%s/raw?ref=master"
+      },
       {
         "https://gitlab.net/eclipse/fooproj/che.git",
-        "https://gitlab.net/eclipse/fooproj/che/-/raw/master/"
+        "https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=master"
       },
       {
         "https://gitlab.net/eclipse/fooproj/-/tree/master/",
-        "https://gitlab.net/eclipse/fooproj/-/raw/master/"
+        "https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=master"
       },
       {
         "https://gitlab.net/eclipse/fooproj/che/-/tree/foobranch/",
-        "https://gitlab.net/eclipse/fooproj/che/-/raw/foobranch/"
+        "https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=foobranch"
       },
       {
         "https://gitlab.net/eclipse/fooproj/che/-/tree/foobranch/subfolder",
-        "https://gitlab.net/eclipse/fooproj/che/-/raw/foobranch/subfolder/"
+        "https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=foobranch"
       },
     };
   }