Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ability to specify custom endpoint url #138

Merged
merged 6 commits into from
Oct 15, 2021
Merged

Add ability to specify custom endpoint url #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6d7bc37
Add ability to specify custom endpoint url
rsandell Oct 7, 2021
fa654a7
Merge branch 'master' into customendpoint
rsandell Oct 7, 2021
6b49249
Update src/test/java/hudson/plugins/s3/MiniIOTest.java
rsandell Oct 8, 2021
9856369
Applying code review suggestions
rsandell Oct 8, 2021
928d444
No need to annotate stapler accessible
rsandell Oct 13, 2021
622dc58
Don't require environment to be set on the agents
rsandell Oct 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>plugin</artifactId>
<version>4.27</version>
<version>4.28</version>
</parent>

<artifactId>s3</artifactId>
Expand All @@ -14,7 +14,7 @@

<properties>
<java.level>8</java.level>
<jenkins.version>2.289.1</jenkins.version>
<jenkins.version>2.289.3</jenkins.version>
</properties>

<developers>
Expand Down Expand Up @@ -75,12 +75,11 @@
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>apache-httpcomponents-client-4-api</artifactId>
<version>4.5.13-1.0</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>copyartifact</artifactId>
<version>1.43</version>
<version>1.46</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.main</groupId>
Expand All @@ -96,27 +95,42 @@
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>structs</artifactId>
<version>1.20</version>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.16.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<dependencyManagement>
<dependencies>
<!-- fix upper bound-->
<dependency>
<groupId>io.jenkins.tools.bom</groupId>
<artifactId>bom-2.289.x</artifactId>
<version>950.v396cb834de1e</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>display-url-api</artifactId>
<version>2.3.5</version>
</dependency>
</dependencies>
</dependencyManagement>


<repositories>
<repository>
<id>repo.jenkins-ci.org</id>
Expand Down
41 changes: 29 additions & 12 deletions src/main/java/hudson/plugins/s3/ClientHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
package hudson.plugins.s3;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.RegionUtils;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import hudson.ProxyConfiguration;
import org.apache.commons.lang.StringUtils;

import java.util.logging.Logger;
import java.util.regex.Pattern;

import javax.annotation.Nonnull;
Expand All @@ -18,23 +22,32 @@ public class ClientHelper {
public final static String DEFAULT_AMAZON_S3_REGION_NAME = System.getProperty(
"hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION",
com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName());
public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("PLUGIN_S3_ENDPOINT"));

public static AmazonS3Client createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy)
public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecate?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did that at first, but as I was editing various calls to it, it felt like a good shortcut to have.

return createClient(accessKey, secretKey, useRole, region, proxy, ENDPOINT);
}

public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy, String customEndpoint)
{
Region awsRegion = getRegionFromString(region);

ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion);

final AmazonS3Client client;
if (useRole) {
client = new AmazonS3Client(clientConfiguration);
} else {
client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration);
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfiguration);

if (!useRole) {
builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
}

client.setRegion(awsRegion);
if (StringUtils.isNotEmpty(customEndpoint)) {
builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customEndpoint, awsRegion.getName()))
.withPathStyleAccessEnabled(true);
} else {
builder = builder.withRegion(awsRegion.getName());
}

return client;
return builder.build();
}

/**
Expand Down Expand Up @@ -73,9 +86,13 @@ private static Region getRegionFromString(@Nullable String regionName) {
@Nonnull
public static ClientConfiguration getClientConfiguration(@Nonnull ProxyConfiguration proxy, @Nonnull Region region) {
final ClientConfiguration clientConfiguration = new ClientConfiguration();

String s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX);

String s3Endpoint;
if (StringUtils.isNotEmpty(ENDPOINT)) {
s3Endpoint = ENDPOINT;
} else {
s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX);
}
Logger.getLogger(ClientHelper.class.getName()).fine(() -> String.format("ENDPOINT: %s", s3Endpoint));
if (shouldUseProxy(proxy, s3Endpoint)) {
clientConfiguration.setProxyHost(proxy.name);
clientConfiguration.setProxyPort(proxy.port);
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/hudson/plugins/s3/S3ArtifactsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import javax.servlet.ServletException;

import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.ResponseHeaderOverrides;
import hudson.Functions;
Expand All @@ -25,7 +25,7 @@

@ExportedBean
public class S3ArtifactsAction implements RunAction2 {
private final Run build; // Compatibility for old versions
private final Run<?,?> build; // Compatibility for old versions
private final String profile;
private final List<FingerprintRecord> artifacts;

Expand Down Expand Up @@ -89,7 +89,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo
for (FingerprintRecord record : artifacts) {
if (record.getArtifact().getName().equals(artifact)) {
final S3Profile s3 = S3BucketPublisher.getProfile(profile);
final AmazonS3Client client = s3.getClient(record.getArtifact().getRegion());
final AmazonS3 client = s3.getClient(record.getArtifact().getRegion());
final String url = getDownloadURL(client, s3.getSignedUrlExpirySeconds(), build, record);
response.sendRedirect2(url);
return;
Expand All @@ -106,7 +106,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo
* download and there's no need for the user to have credentials to
* access S3.
*/
private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) {
private String getDownloadURL(AmazonS3 client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) {
final Destination dest = Destination.newFromRun(run, record.getArtifact());
final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName);
request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000));
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/hudson/plugins/s3/S3BucketPublisher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

import com.amazonaws.AmazonClientException;
import com.amazonaws.regions.Region;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import edu.umd.cs.findbugs.annotations.NonNull;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.AbortException;
import hudson.Extension;
Expand Down Expand Up @@ -201,7 +203,7 @@ public static S3Profile getProfile(String profileName) {
throw new IllegalArgumentException("Can't find profile: " + profileName);
}

@Override
@Override @NonNull
public Collection<? extends Action> getProjectActions(AbstractProject<?, ?> project) {
return ImmutableList.of(new S3ArtifactsProjectAction(project));
}
Expand Down Expand Up @@ -550,7 +552,7 @@ public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter
}

final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME;
final AmazonS3Client client = ClientHelper.createClient(
final AmazonS3 client = ClientHelper.createClient(
checkedAccessKey, checkedSecretKey, useRole, defaultRegion, Jenkins.get().proxy);

try {
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/hudson/plugins/s3/S3CopyArtifact.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ public void perform(@Nonnull Run<?, ?> dst, @Nonnull FilePath targetDir, @Nonnul
}
}

private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console)
private boolean perform(Run<?,?> src, Run<?,?> dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console)
throws IOException, InterruptedException {

final S3ArtifactsAction action = src.getAction(S3ArtifactsAction.class);
Expand All @@ -271,15 +271,15 @@ private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excl

final Map<String, String> fingerprints = Maps.newHashMap();
for(FingerprintRecord record : records) {
final FingerprintMap map = Jenkins.getInstance().getFingerprintMap();
final FingerprintMap map = Jenkins.get().getFingerprintMap();

final Fingerprint f = map.getOrCreate(src, record.getName(), record.getFingerprint());
f.addFor(src);
f.addFor(dst);
fingerprints.put(record.getName(), record.getFingerprint());
}

for (Run r : new Run[]{src, dst}) {
for (Run<?,?> r : new Run<?,?>[]{src, dst}) {
if (r == null) {
continue;
}
Expand All @@ -288,7 +288,7 @@ private boolean perform(Run src, Run<?,?> dst, String includeFilter, String excl
if (fa != null) {
fa.add(fingerprints);
} else {
r.getActions().add(new FingerprintAction(r, fingerprints));
r.addAction(new FingerprintAction(r, fingerprints));
}
}

Expand Down
9 changes: 5 additions & 4 deletions src/main/java/hudson/plugins/s3/S3Profile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package hudson.plugins.s3;

import com.amazonaws.services.s3.AmazonS3;
import hudson.FilePath;

import java.io.IOException;
Expand Down Expand Up @@ -116,7 +117,7 @@ public int getSignedUrlExpirySeconds() {
return signedUrlExpirySeconds;
}

public AmazonS3Client getClient(String region) {
public AmazonS3 getClient(String region) {
return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, getProxy());
}

Expand Down Expand Up @@ -202,7 +203,7 @@ private <T> T invoke(boolean uploadFromSlave, FilePath filePath, MasterSlaveCall
}

public List<String> list(Run build, String bucket) {
final AmazonS3Client s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME);
final AmazonS3 s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME);

final String buildName = build.getDisplayName();
final int buildID = build.getNumber();
Expand Down Expand Up @@ -230,7 +231,7 @@ public List<String> list(Run build, String bucket) {
/**
* Download all artifacts from a given build
*/
public List<FingerprintRecord> downloadAll(Run build,
public List<FingerprintRecord> downloadAll(Run<?,?> build,
final List<FingerprintRecord> artifacts,
final String includeFilter,
final String excludeFilter,
Expand Down Expand Up @@ -286,7 +287,7 @@ private FilePath getFilePath(FilePath targetDir, boolean flatten, String fullNam
public void delete(Run run, FingerprintRecord record) {
final Destination dest = Destination.newFromRun(run, record.getArtifact());
final DeleteObjectRequest req = new DeleteObjectRequest(dest.bucketName, dest.objectName);
final AmazonS3Client client = getClient(record.getArtifact().getRegion());
final AmazonS3 client = getClient(record.getArtifact().getRegion());
client.deleteObject(req);
}

Expand Down
9 changes: 7 additions & 2 deletions src/main/java/hudson/plugins/s3/callable/S3Callable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import hudson.FilePath.FileCallable;
import hudson.ProxyConfiguration;
import hudson.plugins.s3.ClientHelper;
import hudson.util.Secret;
import org.apache.commons.lang.StringUtils;
import org.jenkinsci.remoting.RoleChecker;

import java.io.ObjectStreamException;
import java.util.HashMap;

abstract class S3Callable<T> implements FileCallable<T> {
Expand All @@ -18,6 +21,7 @@ abstract class S3Callable<T> implements FileCallable<T> {
private final boolean useRole;
private final String region;
private final ProxyConfiguration proxy;
private final String customEndpoint;

private static transient HashMap<String, TransferManager> transferManagers = new HashMap<>();

Expand All @@ -27,13 +31,14 @@ abstract class S3Callable<T> implements FileCallable<T> {
this.useRole = useRole;
this.region = region;
this.proxy = proxy;
this.customEndpoint = ClientHelper.ENDPOINT;
}

protected synchronized TransferManager getTransferManager() {
final String uniqueKey = getUniqueKey();
if (transferManagers.get(uniqueKey) == null) {
final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy);
transferManagers.put(uniqueKey, new TransferManager(client));
final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy, customEndpoint);
transferManagers.put(uniqueKey, TransferManagerBuilder.standard().withS3Client(client).build());
}

return transferManagers.get(uniqueKey);
Expand Down
Loading