From 6d7bc375a9af78aab0b5dc6a9d7e01b80e528cd0 Mon Sep 17 00:00:00 2001 From: rsandell Date: Thu, 7 Oct 2021 16:56:36 +0200 Subject: [PATCH 1/5] Add ability to specify custom endpoint url Change to non deprecated API Modernise the plugin pom Added realistic tests --- pom.xml | 43 +++- .../java/hudson/plugins/s3/ClientHelper.java | 37 ++- .../hudson/plugins/s3/S3ArtifactsAction.java | 13 +- .../hudson/plugins/s3/S3BucketPublisher.java | 6 +- .../hudson/plugins/s3/S3CopyArtifact.java | 8 +- .../java/hudson/plugins/s3/S3Profile.java | 9 +- .../plugins/s3/callable/S3Callable.java | 3 +- .../java/hudson/plugins/s3/MiniIOTest.java | 214 ++++++++++++++++++ 8 files changed, 299 insertions(+), 34 deletions(-) create mode 100644 src/test/java/hudson/plugins/s3/MiniIOTest.java diff --git a/pom.xml b/pom.xml index 547a0491..cd158e2b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.jenkins-ci.plugins plugin - 3.54 + 4.25 s3 @@ -14,7 +14,7 @@ 8 - 2.138.4 + 2.277.4 @@ -76,17 +76,16 @@ org.jenkins-ci.plugins apache-httpcomponents-client-4-api - 4.5.10-2.0 org.jenkins-ci.plugins copyartifact - 1.43 + 1.46 org.jenkins-ci.main maven-plugin - 3.4 + 3.12 commons-io @@ -97,10 +96,42 @@ org.jenkins-ci.plugins structs - 1.20 + + + org.testcontainers + testcontainers + 1.16.0 + test + + + org.apache.commons + commons-compress + + + org.slf4j + * + + + + + + io.jenkins.tools.bom + bom-2.277.x + 950.v396cb834de1e + pom + import + + + commons-net + commons-net + 3.8.0 + + + + repo.jenkins-ci.org diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index 8251d2d2..fb0cb2b0 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -1,14 +1,18 @@ package hudson.plugins.s3; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.regions.Regions; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import hudson.ProxyConfiguration; +import org.apache.commons.lang.StringUtils; +import java.util.logging.Logger; import java.util.regex.Pattern; import javax.annotation.Nonnull; @@ -18,23 +22,28 @@ public class ClientHelper { public final static String DEFAULT_AMAZON_S3_REGION_NAME = System.getProperty( "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION", com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); + public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("hudson.plugins.s3.ENDPOINT")); - public static AmazonS3Client createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) { Region awsRegion = getRegionFromString(region); ClientConfiguration clientConfiguration = getClientConfiguration(proxy, awsRegion); - final AmazonS3Client client; - if (useRole) { - client = new AmazonS3Client(clientConfiguration); - } else { - client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), clientConfiguration); + AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfiguration); + + if (!useRole) { + builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); } - client.setRegion(awsRegion); + if (StringUtils.isNotEmpty(ENDPOINT)) { + builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(ENDPOINT, awsRegion.getName())) + .withPathStyleAccessEnabled(true); + } else { + builder = builder.withRegion(awsRegion.getName()); + } - return client; + return builder.build(); } /** @@ -73,9 +82,13 @@ private static Region getRegionFromString(@Nullable String regionName) { @Nonnull public static ClientConfiguration getClientConfiguration(@Nonnull ProxyConfiguration proxy, @Nonnull Region region) { final ClientConfiguration clientConfiguration = new ClientConfiguration(); - - String s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX); - + String s3Endpoint; + if (StringUtils.isNotEmpty(ENDPOINT)) { + s3Endpoint = ENDPOINT; + } else { + s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX); + } + Logger.getLogger(ClientHelper.class.getName()).info("ENDPOINT: " + s3Endpoint); if (shouldUseProxy(proxy, s3Endpoint)) { clientConfiguration.setProxyHost(proxy.name); clientConfiguration.setProxyPort(proxy.port); diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java index 84676e73..d2df4cf0 100644 --- a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java +++ b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java @@ -9,11 +9,13 @@ import javax.servlet.ServletException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; import com.amazonaws.services.s3.model.ResponseHeaderOverrides; import hudson.Functions; import jenkins.model.RunAction2; +import jenkins.security.stapler.StaplerAccessibleType; +import jenkins.security.stapler.StaplerDispatchable; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; @@ -23,9 +25,9 @@ import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; -@ExportedBean +@ExportedBean @StaplerAccessibleType public class S3ArtifactsAction implements RunAction2 { - private final Run build; // Compatibility for old versions + private final Run build; // Compatibility for old versions private final String profile; private final List artifacts; @@ -75,6 +77,7 @@ public List getArtifacts() { return artifacts; } + @StaplerDispatchable public void doDownload(final StaplerRequest request, final StaplerResponse response) throws IOException, ServletException { if (Functions.isArtifactsPermissionEnabled()) { build.getParent().checkPermission(Run.ARTIFACTS); @@ -89,7 +92,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo for (FingerprintRecord record : artifacts) { if (record.getArtifact().getName().equals(artifact)) { final S3Profile s3 = S3BucketPublisher.getProfile(profile); - final AmazonS3Client client = s3.getClient(record.getArtifact().getRegion()); + final AmazonS3 client = s3.getClient(record.getArtifact().getRegion()); final String url = getDownloadURL(client, s3.getSignedUrlExpirySeconds(), build, record); response.sendRedirect2(url); return; @@ -106,7 +109,7 @@ public void doDownload(final StaplerRequest request, final StaplerResponse respo * download and there's no need for the user to have credentials to * access S3. */ - private String getDownloadURL(AmazonS3Client client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { + private String getDownloadURL(AmazonS3 client, int signedUrlExpirySeconds, Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(dest.bucketName, dest.objectName); request.setExpiration(new Date(System.currentTimeMillis() + signedUrlExpirySeconds*1000)); diff --git a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java index e96b592f..ceabe70a 100644 --- a/src/main/java/hudson/plugins/s3/S3BucketPublisher.java +++ b/src/main/java/hudson/plugins/s3/S3BucketPublisher.java @@ -2,10 +2,12 @@ import com.amazonaws.AmazonClientException; import com.amazonaws.regions.Region; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.AbortException; import hudson.Extension; @@ -201,7 +203,7 @@ public static S3Profile getProfile(String profileName) { throw new IllegalArgumentException("Can't find profile: " + profileName); } - @Override + @Override @NonNull public Collection getProjectActions(AbstractProject project) { return ImmutableList.of(new S3ArtifactsProjectAction(project)); } @@ -550,7 +552,7 @@ public FormValidation doLoginCheck(@QueryParameter String name, @QueryParameter } final String defaultRegion = ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME; - final AmazonS3Client client = ClientHelper.createClient( + final AmazonS3 client = ClientHelper.createClient( checkedAccessKey, checkedSecretKey, useRole, defaultRegion, Jenkins.get().proxy); try { diff --git a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java index 1ab7d8bd..af4f214e 100644 --- a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java +++ b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java @@ -250,7 +250,7 @@ public void perform(@Nonnull Run dst, @Nonnull FilePath targetDir, @Nonnul } } - private boolean perform(Run src, Run dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console) + private boolean perform(Run src, Run dst, String includeFilter, String excludeFilter, FilePath targetDir, PrintStream console) throws IOException, InterruptedException { final S3ArtifactsAction action = src.getAction(S3ArtifactsAction.class); @@ -271,7 +271,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl final Map fingerprints = Maps.newHashMap(); for(FingerprintRecord record : records) { - final FingerprintMap map = Jenkins.getInstance().getFingerprintMap(); + final FingerprintMap map = Jenkins.get().getFingerprintMap(); final Fingerprint f = map.getOrCreate(src, record.getName(), record.getFingerprint()); f.addFor(src); @@ -279,7 +279,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl fingerprints.put(record.getName(), record.getFingerprint()); } - for (Run r : new Run[]{src, dst}) { + for (Run r : new Run[]{src, dst}) { if (r == null) { continue; } @@ -288,7 +288,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String excl if (fa != null) { fa.add(fingerprints); } else { - r.getActions().add(new FingerprintAction(r, fingerprints)); + r.addAction(new FingerprintAction(r, fingerprints)); } } diff --git a/src/main/java/hudson/plugins/s3/S3Profile.java b/src/main/java/hudson/plugins/s3/S3Profile.java index 9a3d1fa7..6095e8da 100644 --- a/src/main/java/hudson/plugins/s3/S3Profile.java +++ b/src/main/java/hudson/plugins/s3/S3Profile.java @@ -1,5 +1,6 @@ package hudson.plugins.s3; +import com.amazonaws.services.s3.AmazonS3; import hudson.FilePath; import java.io.IOException; @@ -116,7 +117,7 @@ public int getSignedUrlExpirySeconds() { return signedUrlExpirySeconds; } - public AmazonS3Client getClient(String region) { + public AmazonS3 getClient(String region) { return ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, getProxy()); } @@ -202,7 +203,7 @@ private T invoke(boolean uploadFromSlave, FilePath filePath, MasterSlaveCall } public List list(Run build, String bucket) { - final AmazonS3Client s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); + final AmazonS3 s3client = getClient(ClientHelper.DEFAULT_AMAZON_S3_REGION_NAME); final String buildName = build.getDisplayName(); final int buildID = build.getNumber(); @@ -230,7 +231,7 @@ public List list(Run build, String bucket) { /** * Download all artifacts from a given build */ - public List downloadAll(Run build, + public List downloadAll(Run build, final List artifacts, final String includeFilter, final String excludeFilter, @@ -286,7 +287,7 @@ private FilePath getFilePath(FilePath targetDir, boolean flatten, String fullNam public void delete(Run run, FingerprintRecord record) { final Destination dest = Destination.newFromRun(run, record.getArtifact()); final DeleteObjectRequest req = new DeleteObjectRequest(dest.bucketName, dest.objectName); - final AmazonS3Client client = getClient(record.getArtifact().getRegion()); + final AmazonS3 client = getClient(record.getArtifact().getRegion()); client.deleteObject(req); } diff --git a/src/main/java/hudson/plugins/s3/callable/S3Callable.java b/src/main/java/hudson/plugins/s3/callable/S3Callable.java index 0916366a..68e3c1de 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3Callable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3Callable.java @@ -2,6 +2,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import hudson.FilePath.FileCallable; import hudson.ProxyConfiguration; import hudson.plugins.s3.ClientHelper; @@ -33,7 +34,7 @@ protected synchronized TransferManager getTransferManager() { final String uniqueKey = getUniqueKey(); if (transferManagers.get(uniqueKey) == null) { final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy); - transferManagers.put(uniqueKey, new TransferManager(client)); + transferManagers.put(uniqueKey, TransferManagerBuilder.standard().withS3Client(client).build()); } return transferManagers.get(uniqueKey); diff --git a/src/test/java/hudson/plugins/s3/MiniIOTest.java b/src/test/java/hudson/plugins/s3/MiniIOTest.java new file mode 100644 index 00000000..269be177 --- /dev/null +++ b/src/test/java/hudson/plugins/s3/MiniIOTest.java @@ -0,0 +1,214 @@ +/* + * The MIT License + * + * Copyright (c) 2021, CloudBees Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package hudson.plugins.s3; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import hudson.EnvVars; +import hudson.ExtensionList; +import hudson.FilePath; +import hudson.Launcher; +import hudson.model.AbstractBuild; +import hudson.model.BuildListener; +import hudson.model.FreeStyleProject; +import hudson.model.Label; +import hudson.plugins.copyartifact.LastCompletedBuildSelector; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.CreateFileBuilder; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.RealJenkinsRule; +import org.jvnet.hudson.test.TestBuilder; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; + +import static org.junit.Assert.assertTrue; + +public class MiniIOTest { + + private static final String ACCESS_KEY = "supersecure"; + private static final String SECRET_KEY = "donttell"; + private static final String CONTAINER_NAME = "jenkins"; + private static final String CONTAINER_PREFIX = "ci/"; + private static final String REGION = "local"; + public static final String FILE_CONTENT = "Hello World"; + + private static GenericContainer minioServer; + private static String minioServiceEndpoint; + + @Rule + public RealJenkinsRule rr = new RealJenkinsRule().javaOptions("-Xmx256m", + "-Dhudson.plugins.s3.DEFAULT_AMAZON_S3_REGION=local", "-Dhudson.plugins.s3.ENDPOINT=http://" + minioServiceEndpoint); + + @BeforeClass + public static void setUpClass() throws Exception { + try { + DockerClientFactory.instance().client(); + } catch (Exception x) { + Assume.assumeNoException("does not look like Docker is available", x); + } + int port = 9000; + minioServer = new GenericContainer("minio/minio") + .withEnv("MINIO_ACCESS_KEY", ACCESS_KEY) + .withEnv("MINIO_SECRET_KEY", SECRET_KEY) + .withCommand("server /data") + .withExposedPorts(port) + .waitingFor(new HttpWaitStrategy() + .forPath("/minio/health/ready") + .forPort(port) + .withStartupTimeout(Duration.ofSeconds(10))); + minioServer.start(); + + Integer mappedPort = minioServer.getFirstMappedPort(); + Testcontainers.exposeHostPorts(mappedPort); + minioServiceEndpoint = String.format("%s:%s", minioServer.getContainerIpAddress(), mappedPort); + + final AmazonS3 client = AmazonS3ClientBuilder.standard() + //.withRegion("local") + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY))) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://" + minioServiceEndpoint, "us-east-1")) + //.withForceGlobalBucketAccessEnabled(true) + .withPathStyleAccessEnabled(true) + .build(); + + if (!client.doesBucketExistV2("test")) { + client.createBucket("test"); + assertTrue(client.doesBucketExistV2("test")); + } + } + + @AfterClass + public static void shutDownClass() { + if (minioServer != null && minioServer.isRunning()) { + minioServer.stop(); + } + } + + @Test + public void testS3BucketPublisher() throws Throwable { + final String endpoint = minioServiceEndpoint; + rr.then(r -> { + + /*r.jenkins.setLabelString("work"); //Able to debug when running on the controller but not an agent + r.jenkins.setNumExecutors(1);*/ + r.createOnlineSlave(Label.get("work"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); + + createProfile(); + + + createAndRunPublisher(r); + }); + } + + private static void createAndRunPublisher(final JenkinsRule r) throws Exception { + final FreeStyleProject job = r.createFreeStyleProject("publisherJob"); + job.setAssignedLabel(Label.get("work")); + job.getBuildersList().add(new CreateFileBuilder("test.txt", FILE_CONTENT)); + job.getPublishersList().add(new S3BucketPublisher("Local", + Collections.singletonList(new Entry("test", + "test.txt", + "", + null, + "", + false, + true, + true, + false, + false, + false, + true, + false, + Collections.emptyList())), + Collections.emptyList(), + true, + "FINE", + null, false)); + r.buildAndAssertSuccess(job); + } + + private static void createProfile() { + final S3BucketPublisher.DescriptorImpl descriptor = ExtensionList.lookup(S3BucketPublisher.DescriptorImpl.class).get(0); + descriptor.replaceProfiles(Collections.singletonList(new S3Profile( + "Local", + ACCESS_KEY, SECRET_KEY, + false, + 10000, + "", "", "", "", true))); + } + + @Test + public void testS3CopyArtifact() throws Throwable { + final String endpoint = minioServiceEndpoint; + rr.then(r -> { + r.createOnlineSlave(Label.get("work"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); + r.createOnlineSlave(Label.get("copy"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); + + createProfile(); + createAndRunPublisher(r); + + FreeStyleProject job = r.createFreeStyleProject("copierJob"); + job.setAssignedLabel(Label.get("copy")); + job.getBuildersList().add(new S3CopyArtifact( + "publisherJob", + new LastCompletedBuildSelector(), + "*.txt", + "", + "", + false, + false + )); + job.getBuildersList().add(new VerifyFileBuilder()); + r.buildAndAssertSuccess(job); + }); + } + + public static class VerifyFileBuilder extends TestBuilder { + @Override + public boolean perform(final AbstractBuild build, final Launcher launcher, final BuildListener listener) throws InterruptedException, IOException { + final FilePath child = build.getWorkspace().child("test.txt"); + if (!child.exists()) { + listener.error("No test.txt in workspace!"); + return false; + } + final String s = child.readToString(); + if (!FILE_CONTENT.equals(s)) { + listener.error("Wrong file content! Expected \"" + FILE_CONTENT + "\" but was \"" + s + "\""); + return false; + } + return true; + } + } +} From 6b49249fdde80c0cb77d21fbd40b6aa14dd36dab Mon Sep 17 00:00:00 2001 From: Robert Sandell Date: Fri, 8 Oct 2021 10:30:23 +0200 Subject: [PATCH 2/5] Update src/test/java/hudson/plugins/s3/MiniIOTest.java Co-authored-by: Jesse Glick --- src/test/java/hudson/plugins/s3/MiniIOTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/hudson/plugins/s3/MiniIOTest.java b/src/test/java/hudson/plugins/s3/MiniIOTest.java index 269be177..605f76b7 100644 --- a/src/test/java/hudson/plugins/s3/MiniIOTest.java +++ b/src/test/java/hudson/plugins/s3/MiniIOTest.java @@ -57,7 +57,7 @@ import static org.junit.Assert.assertTrue; -public class MiniIOTest { +public class MinIOTest { private static final String ACCESS_KEY = "supersecure"; private static final String SECRET_KEY = "donttell"; From 9856369cddfa3ed8fb1664126671c38232b2ac1b Mon Sep 17 00:00:00 2001 From: rsandell Date: Fri, 8 Oct 2021 10:46:49 +0200 Subject: [PATCH 3/5] Applying code review suggestions --- .../java/hudson/plugins/s3/ClientHelper.java | 4 +-- .../hudson/plugins/s3/S3CopyArtifact.java | 2 +- .../s3/{MiniIOTest.java => MinIOTest.java} | 25 +++++++------------ 3 files changed, 12 insertions(+), 19 deletions(-) rename src/test/java/hudson/plugins/s3/{MiniIOTest.java => MinIOTest.java} (92%) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index fb0cb2b0..741e655b 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -22,7 +22,7 @@ public class ClientHelper { public final static String DEFAULT_AMAZON_S3_REGION_NAME = System.getProperty( "hudson.plugins.s3.DEFAULT_AMAZON_S3_REGION", com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); - public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("hudson.plugins.s3.ENDPOINT")); + public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("PLUGIN_S3_ENDPOINT")); public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) { @@ -88,7 +88,7 @@ public static ClientConfiguration getClientConfiguration(@Nonnull ProxyConfigura } else { s3Endpoint = region.getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX); } - Logger.getLogger(ClientHelper.class.getName()).info("ENDPOINT: " + s3Endpoint); + Logger.getLogger(ClientHelper.class.getName()).fine(() -> String.format("ENDPOINT: %s", s3Endpoint)); if (shouldUseProxy(proxy, s3Endpoint)) { clientConfiguration.setProxyHost(proxy.name); clientConfiguration.setProxyPort(proxy.port); diff --git a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java index af4f214e..5a498846 100644 --- a/src/main/java/hudson/plugins/s3/S3CopyArtifact.java +++ b/src/main/java/hudson/plugins/s3/S3CopyArtifact.java @@ -279,7 +279,7 @@ private boolean perform(Run src, Run dst, String includeFilter, String fingerprints.put(record.getName(), record.getFingerprint()); } - for (Run r : new Run[]{src, dst}) { + for (Run r : new Run[]{src, dst}) { if (r == null) { continue; } diff --git a/src/test/java/hudson/plugins/s3/MiniIOTest.java b/src/test/java/hudson/plugins/s3/MinIOTest.java similarity index 92% rename from src/test/java/hudson/plugins/s3/MiniIOTest.java rename to src/test/java/hudson/plugins/s3/MinIOTest.java index 605f76b7..cb1ddaf2 100644 --- a/src/test/java/hudson/plugins/s3/MiniIOTest.java +++ b/src/test/java/hudson/plugins/s3/MinIOTest.java @@ -55,6 +55,7 @@ import java.time.Duration; import java.util.Collections; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class MinIOTest { @@ -71,7 +72,7 @@ public class MinIOTest { @Rule public RealJenkinsRule rr = new RealJenkinsRule().javaOptions("-Xmx256m", - "-Dhudson.plugins.s3.DEFAULT_AMAZON_S3_REGION=local", "-Dhudson.plugins.s3.ENDPOINT=http://" + minioServiceEndpoint); + "-Dhudson.plugins.s3.ENDPOINT=http://" + minioServiceEndpoint); @BeforeClass public static void setUpClass() throws Exception { @@ -121,14 +122,10 @@ public static void shutDownClass() { public void testS3BucketPublisher() throws Throwable { final String endpoint = minioServiceEndpoint; rr.then(r -> { - /*r.jenkins.setLabelString("work"); //Able to debug when running on the controller but not an agent r.jenkins.setNumExecutors(1);*/ - r.createOnlineSlave(Label.get("work"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); - + r.createOnlineSlave(Label.get("work"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); createProfile(); - - createAndRunPublisher(r); }); } @@ -173,8 +170,8 @@ private static void createProfile() { public void testS3CopyArtifact() throws Throwable { final String endpoint = minioServiceEndpoint; rr.then(r -> { - r.createOnlineSlave(Label.get("work"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); - r.createOnlineSlave(Label.get("copy"), new EnvVars("hudson.plugins.s3.ENDPOINT", "http://" + endpoint)); + r.createOnlineSlave(Label.get("work"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); + r.createOnlineSlave(Label.get("copy"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); createProfile(); createAndRunPublisher(r); @@ -199,15 +196,11 @@ public static class VerifyFileBuilder extends TestBuilder { @Override public boolean perform(final AbstractBuild build, final Launcher launcher, final BuildListener listener) throws InterruptedException, IOException { final FilePath child = build.getWorkspace().child("test.txt"); - if (!child.exists()) { - listener.error("No test.txt in workspace!"); - return false; - } + assertTrue("No test.txt in workspace!", child.exists()); + final String s = child.readToString(); - if (!FILE_CONTENT.equals(s)) { - listener.error("Wrong file content! Expected \"" + FILE_CONTENT + "\" but was \"" + s + "\""); - return false; - } + assertEquals(FILE_CONTENT, s); + return true; } } From 928d4442639bacd1d28d6b26b49ceeabd222867f Mon Sep 17 00:00:00 2001 From: rsandell Date: Wed, 13 Oct 2021 16:31:13 +0200 Subject: [PATCH 4/5] No need to annotate stapler accessible --- src/main/java/hudson/plugins/s3/S3ArtifactsAction.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java index d2df4cf0..cacdcab1 100644 --- a/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java +++ b/src/main/java/hudson/plugins/s3/S3ArtifactsAction.java @@ -14,8 +14,6 @@ import com.amazonaws.services.s3.model.ResponseHeaderOverrides; import hudson.Functions; import jenkins.model.RunAction2; -import jenkins.security.stapler.StaplerAccessibleType; -import jenkins.security.stapler.StaplerDispatchable; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; @@ -25,7 +23,7 @@ import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; -@ExportedBean @StaplerAccessibleType +@ExportedBean public class S3ArtifactsAction implements RunAction2 { private final Run build; // Compatibility for old versions private final String profile; @@ -77,7 +75,6 @@ public List getArtifacts() { return artifacts; } - @StaplerDispatchable public void doDownload(final StaplerRequest request, final StaplerResponse response) throws IOException, ServletException { if (Functions.isArtifactsPermissionEnabled()) { build.getParent().checkPermission(Run.ARTIFACTS); From 622dc581a6cc65d4eb7a5cea99259cdc5cce2f6b Mon Sep 17 00:00:00 2001 From: rsandell Date: Wed, 13 Oct 2021 17:06:16 +0200 Subject: [PATCH 5/5] Don't require environment to be set on the agents --- src/main/java/hudson/plugins/s3/ClientHelper.java | 10 +++++++--- .../java/hudson/plugins/s3/callable/S3Callable.java | 6 +++++- src/test/java/hudson/plugins/s3/MinIOTest.java | 8 +++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/hudson/plugins/s3/ClientHelper.java b/src/main/java/hudson/plugins/s3/ClientHelper.java index 741e655b..318288e8 100644 --- a/src/main/java/hudson/plugins/s3/ClientHelper.java +++ b/src/main/java/hudson/plugins/s3/ClientHelper.java @@ -24,7 +24,11 @@ public class ClientHelper { com.amazonaws.services.s3.model.Region.US_Standard.toAWSRegion().getName()); public static final String ENDPOINT = System.getProperty("hudson.plugins.s3.ENDPOINT", System.getenv("PLUGIN_S3_ENDPOINT")); - public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy) { + return createClient(accessKey, secretKey, useRole, region, proxy, ENDPOINT); + } + + public static AmazonS3 createClient(String accessKey, String secretKey, boolean useRole, String region, ProxyConfiguration proxy, String customEndpoint) { Region awsRegion = getRegionFromString(region); @@ -36,8 +40,8 @@ public static AmazonS3 createClient(String accessKey, String secretKey, boolean builder = builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))); } - if (StringUtils.isNotEmpty(ENDPOINT)) { - builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(ENDPOINT, awsRegion.getName())) + if (StringUtils.isNotEmpty(customEndpoint)) { + builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customEndpoint, awsRegion.getName())) .withPathStyleAccessEnabled(true); } else { builder = builder.withRegion(awsRegion.getName()); diff --git a/src/main/java/hudson/plugins/s3/callable/S3Callable.java b/src/main/java/hudson/plugins/s3/callable/S3Callable.java index 68e3c1de..5dfa6ab8 100644 --- a/src/main/java/hudson/plugins/s3/callable/S3Callable.java +++ b/src/main/java/hudson/plugins/s3/callable/S3Callable.java @@ -7,8 +7,10 @@ import hudson.ProxyConfiguration; import hudson.plugins.s3.ClientHelper; import hudson.util.Secret; +import org.apache.commons.lang.StringUtils; import org.jenkinsci.remoting.RoleChecker; +import java.io.ObjectStreamException; import java.util.HashMap; abstract class S3Callable implements FileCallable { @@ -19,6 +21,7 @@ abstract class S3Callable implements FileCallable { private final boolean useRole; private final String region; private final ProxyConfiguration proxy; + private final String customEndpoint; private static transient HashMap transferManagers = new HashMap<>(); @@ -28,12 +31,13 @@ abstract class S3Callable implements FileCallable { this.useRole = useRole; this.region = region; this.proxy = proxy; + this.customEndpoint = ClientHelper.ENDPOINT; } protected synchronized TransferManager getTransferManager() { final String uniqueKey = getUniqueKey(); if (transferManagers.get(uniqueKey) == null) { - final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy); + final AmazonS3 client = ClientHelper.createClient(accessKey, Secret.toString(secretKey), useRole, region, proxy, customEndpoint); transferManagers.put(uniqueKey, TransferManagerBuilder.standard().withS3Client(client).build()); } diff --git a/src/test/java/hudson/plugins/s3/MinIOTest.java b/src/test/java/hudson/plugins/s3/MinIOTest.java index cb1ddaf2..aede518c 100644 --- a/src/test/java/hudson/plugins/s3/MinIOTest.java +++ b/src/test/java/hudson/plugins/s3/MinIOTest.java @@ -98,10 +98,8 @@ public static void setUpClass() throws Exception { minioServiceEndpoint = String.format("%s:%s", minioServer.getContainerIpAddress(), mappedPort); final AmazonS3 client = AmazonS3ClientBuilder.standard() - //.withRegion("local") .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY))) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://" + minioServiceEndpoint, "us-east-1")) - //.withForceGlobalBucketAccessEnabled(true) .withPathStyleAccessEnabled(true) .build(); @@ -124,7 +122,7 @@ public void testS3BucketPublisher() throws Throwable { rr.then(r -> { /*r.jenkins.setLabelString("work"); //Able to debug when running on the controller but not an agent r.jenkins.setNumExecutors(1);*/ - r.createOnlineSlave(Label.get("work"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); + r.createOnlineSlave(Label.get("work")); createProfile(); createAndRunPublisher(r); }); @@ -170,8 +168,8 @@ private static void createProfile() { public void testS3CopyArtifact() throws Throwable { final String endpoint = minioServiceEndpoint; rr.then(r -> { - r.createOnlineSlave(Label.get("work"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); - r.createOnlineSlave(Label.get("copy"), new EnvVars("PLUGIN_S3_ENDPOINT", "http://" + endpoint)); + r.createOnlineSlave(Label.get("work")); + r.createOnlineSlave(Label.get("copy")); createProfile(); createAndRunPublisher(r);