diff --git a/pom.xml b/pom.xml index 88b7eaad..1281c5e2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.jenkins-ci.plugins plugin - 4.27 + 4.28 s3 @@ -14,7 +14,7 @@ 8 - 2.289.1 + 2.289.3 @@ -75,12 +75,11 @@ org.jenkins-ci.plugins apache-httpcomponents-client-4-api - 4.5.13-1.0 org.jenkins-ci.plugins copyartifact - 1.43 + 1.46 org.jenkins-ci.main @@ -96,27 +95,42 @@ org.jenkins-ci.plugins structs - 1.20 + + + org.testcontainers + testcontainers + 1.16.0 + test + + + org.apache.commons + commons-compress + + + org.slf4j + * + + - + + io.jenkins.tools.bom + bom-2.289.x + 950.v396cb834de1e + pom + import + commons-net commons-net 3.8.0 - - org.jenkins-ci.plugins - display-url-api - 2.3.5 - - repo.jenkins-ci.org diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index 8251d2d2..318288e8 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -1,14 +1,18 @@ package hudson.plugins.s3; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import hudson.ProxyConfiguration; +import org.apache.commons.lang.StringUtils; +import java.util.logging.Logger; import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -18,23 +22,32 @@ public class ClientHelper { public final static String DEFAULT_AMAZON_S3_REGION_NAME = System.getProperty( "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION", com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); + public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("PLUGIN_S3_ENDPOINT")); - public static AmazonS3Client createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) { + return createClient(accessKey, secretKey, useRole, region, proxy, ENDPOINT); + } + + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy, String customEndpoint) { Region awsRegion = getRegionFromString(region); ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion); - final AmazonS3Client client; - if (useRole) { - client = new AmazonS3Client(clientConfiguration); - } else { - client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration); + AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfiguration); + + if (!useRole) { + builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); } - client.setRegion(awsRegion); + if (StringUtils.isNotEmpty(customEndpoint)) { + builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customEndpoint, awsRegion.getName())) + .withPathStyleAccessEnabled(true); + } else { + builder = builder.withRegion(awsRegion.getName()); + } - return client; + return builder.build(); } /** @@ -73,9 +86,13 @@ private static Region getRegionFromString(@Nullable String regionName) { @Nonnull public static ClientConfiguration getClientConfiguration(@Nonnull ProxyConfiguration proxy, @Nonnull Region region) { final ClientConfiguration clientConfiguration = new ClientConfiguration(); - - String s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX); - + String s3Endpoint; + if (StringUtils.isNotEmpty(ENDPOINT)) { + s3Endpoint = ENDPOINT; + } else { + s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX); + } + Logger.getLogger(ClientHelper.class.getName()).fine(() -> String.format("ENDPOINT: %s", s3Endpoint)); if (shouldUseProxy(proxy, s3Endpoint)) { clientConfiguration.setProxyHost(proxy.name); clientConfiguration.setProxyPort(proxy.port); diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java index 84676e73..cacdcab1 100644 --- a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java +++ b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java @@ -9,7 +9,7 @@ import javax.servlet.ServletException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.ResponseHeaderOverrides; import hudson.Functions; @@ -25,7 +25,7 @@ @ExportedBean public class S3ArtifactsAction implements RunAction2 { - private final Run build; // Compatibility for old versions + private final Run build; // Compatibility for old versions private final String profile; private final List artifacts; @@ -89,7 +89,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo for (FingerprintRecord record : artifacts) { if (record.getArtifact().getName().equals(artifact)) { final S3Profile s3 = S3BucketPublisher.getProfile(profile); - final AmazonS3Client client = s3.getClient(record.getArtifact().getRegion()); + final AmazonS3 client = s3.getClient(record.getArtifact().getRegion()); final String url = getDownloadURL(client, s3.getSignedUrlExpirySeconds(), build, record); response.sendRedirect2(url); return; @@ -106,7 +106,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo * download and there's no need for the user to have credentials to * access S3. */ - private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { + private String getDownloadURL(AmazonS3 client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName); request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000)); diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index e96b592f..ceabe70a 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -2,10 +2,12 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.regions.Region; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.AbortException; import hudson.Extension; @@ -201,7 +203,7 @@ public static S3Profile getProfile(String profileName) { throw new IllegalArgumentException("Can't find profile: " + profileName); } - @Override + @Override @NonNull public Collection getProjectActions(AbstractProject project) { return ImmutableList.of(new S3ArtifactsProjectAction(project)); } @@ -550,7 +552,7 @@ public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter } final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; - final AmazonS3Client client = ClientHelper.createClient( + final AmazonS3 client = ClientHelper.createClient( checkedAccessKey, checkedSecretKey, useRole, defaultRegion, Jenkins.get().proxy); try { diff --git a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java index 1ab7d8bd..5a498846 100644 --- a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java +++ b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java @@ -250,7 +250,7 @@ public void perform(@Nonnull Run dst, @Nonnull FilePath targetDir, @Nonnul } } - private boolean perform(Run src, Run dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console) + private boolean perform(Run src, Run dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console) throws IOException, InterruptedException { final S3ArtifactsAction action = src.getAction(S3ArtifactsAction.class); @@ -271,7 +271,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl final Map fingerprints = Maps.newHashMap(); for(FingerprintRecord record : records) { - final FingerprintMap map = Jenkins.getInstance().getFingerprintMap(); + final FingerprintMap map = Jenkins.get().getFingerprintMap(); final Fingerprint f = map.getOrCreate(src, record.getName(), record.getFingerprint()); f.addFor(src); @@ -279,7 +279,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl fingerprints.put(record.getName(), record.getFingerprint()); } - for (Run r : new Run[]{src, dst}) { + for (Run r : new Run[]{src, dst}) { if (r == null) { continue; } @@ -288,7 +288,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl if (fa != null) { fa.add(fingerprints); } else { - r.getActions().add(new FingerprintAction(r, fingerprints)); + r.addAction(new FingerprintAction(r, fingerprints)); } } diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 9a3d1fa7..6095e8da 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -1,5 +1,6 @@ package hudson.plugins.s3; +import com.amazonaws.services.s3.AmazonS3; import hudson.FilePath; import java.io.IOException; @@ -116,7 +117,7 @@ public int getSignedUrlExpirySeconds() { return signedUrlExpirySeconds; } - public AmazonS3Client getClient(String region) { + public AmazonS3 getClient(String region) { return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, getProxy()); } @@ -202,7 +203,7 @@ private T invoke(boolean uploadFromSlave, FilePath filePath, MasterSlaveCall } public List list(Run build, String bucket) { - final AmazonS3Client s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); + final AmazonS3 s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); final String buildName = build.getDisplayName(); final int buildID = build.getNumber(); @@ -230,7 +231,7 @@ public List list(Run build, String bucket) { /** * Download all artifacts from a given build */ - public List downloadAll(Run build, + public List downloadAll(Run build, final List artifacts, final String includeFilter, final String excludeFilter, @@ -286,7 +287,7 @@ private FilePath getFilePath(FilePath targetDir, boolean flatten, String fullNam public void delete(Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final DeleteObjectRequest req = new DeleteObjectRequest(dest.bucketName, dest.objectName); - final AmazonS3Client client = getClient(record.getArtifact().getRegion()); + final AmazonS3 client = getClient(record.getArtifact().getRegion()); client.deleteObject(req); } diff --git a/src/main/java/hudson/plugins/s3/callable/S3Callable.java b/src/main/java/hudson/plugins/s3/callable/S3Callable.java index 0916366a..5dfa6ab8 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3Callable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3Callable.java @@ -2,12 +2,15 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import hudson.FilePath.FileCallable; import hudson.ProxyConfiguration; import hudson.plugins.s3.ClientHelper; import hudson.util.Secret; +import org.apache.commons.lang.StringUtils; import org.jenkinsci.remoting.RoleChecker; +import java.io.ObjectStreamException; import java.util.HashMap; abstract class S3Callable implements FileCallable { @@ -18,6 +21,7 @@ abstract class S3Callable implements FileCallable { private final boolean useRole; private final String region; private final ProxyConfiguration proxy; + private final String customEndpoint; private static transient HashMap transferManagers = new HashMap<>(); @@ -27,13 +31,14 @@ abstract class S3Callable implements FileCallable { this.useRole = useRole; this.region = region; this.proxy = proxy; + this.customEndpoint = ClientHelper.ENDPOINT; } protected synchronized TransferManager getTransferManager() { final String uniqueKey = getUniqueKey(); if (transferManagers.get(uniqueKey) == null) { - final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy); - transferManagers.put(uniqueKey, new TransferManager(client)); + final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy, customEndpoint); + transferManagers.put(uniqueKey, TransferManagerBuilder.standard().withS3Client(client).build()); } return transferManagers.get(uniqueKey); diff --git a/src/test/java/hudson/plugins/s3/MinIOTest.java b/src/test/java/hudson/plugins/s3/MinIOTest.java new file mode 100644 index 00000000..aede518c --- /dev/null +++ b/src/test/java/hudson/plugins/s3/MinIOTest.java @@ -0,0 +1,205 @@ +/* + * The MIT License + * + * Copyright (c) 2021, CloudBees Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package hudson.plugins.s3; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import hudson.EnvVars; +import hudson.ExtensionList; +import hudson.FilePath; +import hudson.Launcher; +import hudson.model.AbstractBuild; +import hudson.model.BuildListener; +import hudson.model.FreeStyleProject; +import hudson.model.Label; +import hudson.plugins.copyartifact.LastCompletedBuildSelector; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.CreateFileBuilder; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.RealJenkinsRule; +import org.jvnet.hudson.test.TestBuilder; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MinIOTest { + + private static final String ACCESS_KEY = "supersecure"; + private static final String SECRET_KEY = "donttell"; + private static final String CONTAINER_NAME = "jenkins"; + private static final String CONTAINER_PREFIX = "ci/"; + private static final String REGION = "local"; + public static final String FILE_CONTENT = "Hello World"; + + private static GenericContainer minioServer; + private static String minioServiceEndpoint; + + @Rule + public RealJenkinsRule rr = new RealJenkinsRule().javaOptions("-Xmx256m", + "-Dhudson.plugins.s3.ENDPOINT=http://" + minioServiceEndpoint); + + @BeforeClass + public static void setUpClass() throws Exception { + try { + DockerClientFactory.instance().client(); + } catch (Exception x) { + Assume.assumeNoException("does not look like Docker is available", x); + } + int port = 9000; + minioServer = new GenericContainer("minio/minio") + .withEnv("MINIO_ACCESS_KEY", ACCESS_KEY) + .withEnv("MINIO_SECRET_KEY", SECRET_KEY) + .withCommand("server /data") + .withExposedPorts(port) + .waitingFor(new HttpWaitStrategy() + .forPath("/minio/health/ready") + .forPort(port) + .withStartupTimeout(Duration.ofSeconds(10))); + minioServer.start(); + + Integer mappedPort = minioServer.getFirstMappedPort(); + Testcontainers.exposeHostPorts(mappedPort); + minioServiceEndpoint = String.format("%s:%s", minioServer.getContainerIpAddress(), mappedPort); + + final AmazonS3 client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY))) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://" + minioServiceEndpoint, "us-east-1")) + .withPathStyleAccessEnabled(true) + .build(); + + if (!client.doesBucketExistV2("test")) { + client.createBucket("test"); + assertTrue(client.doesBucketExistV2("test")); + } + } + + @AfterClass + public static void shutDownClass() { + if (minioServer != null && minioServer.isRunning()) { + minioServer.stop(); + } + } + + @Test + public void testS3BucketPublisher() throws Throwable { + final String endpoint = minioServiceEndpoint; + rr.then(r -> { + /*r.jenkins.setLabelString("work"); //Able to debug when running on the controller but not an agent + r.jenkins.setNumExecutors(1);*/ + r.createOnlineSlave(Label.get("work")); + createProfile(); + createAndRunPublisher(r); + }); + } + + private static void createAndRunPublisher(final JenkinsRule r) throws Exception { + final FreeStyleProject job = r.createFreeStyleProject("publisherJob"); + job.setAssignedLabel(Label.get("work")); + job.getBuildersList().add(new CreateFileBuilder("test.txt", FILE_CONTENT)); + job.getPublishersList().add(new S3BucketPublisher("Local", + Collections.singletonList(new Entry("test", + "test.txt", + "", + null, + "", + false, + true, + true, + false, + false, + false, + true, + false, + Collections.emptyList())), + Collections.emptyList(), + true, + "FINE", + null, false)); + r.buildAndAssertSuccess(job); + } + + private static void createProfile() { + final S3BucketPublisher.DescriptorImpl descriptor = ExtensionList.lookup(S3BucketPublisher.DescriptorImpl.class).get(0); + descriptor.replaceProfiles(Collections.singletonList(new S3Profile( + "Local", + ACCESS_KEY, SECRET_KEY, + false, + 10000, + "", "", "", "", true))); + } + + @Test + public void testS3CopyArtifact() throws Throwable { + final String endpoint = minioServiceEndpoint; + rr.then(r -> { + r.createOnlineSlave(Label.get("work")); + r.createOnlineSlave(Label.get("copy")); + + createProfile(); + createAndRunPublisher(r); + + FreeStyleProject job = r.createFreeStyleProject("copierJob"); + job.setAssignedLabel(Label.get("copy")); + job.getBuildersList().add(new S3CopyArtifact( + "publisherJob", + new LastCompletedBuildSelector(), + "*.txt", + "", + "", + false, + false + )); + job.getBuildersList().add(new VerifyFileBuilder()); + r.buildAndAssertSuccess(job); + }); + } + + public static class VerifyFileBuilder extends TestBuilder { + @Override + public boolean perform(final AbstractBuild build, final Launcher launcher, final BuildListener listener) throws InterruptedException, IOException { + final FilePath child = build.getWorkspace().child("test.txt"); + assertTrue("No test.txt in workspace!", child.exists()); + + final String s = child.readToString(); + assertEquals(FILE_CONTENT, s); + + return true; + } + } +}