From 86678e8afe68bdb95dd1df2386345ad676decd60 Mon Sep 17 00:00:00 2001 From: Joe Eugene Date: Tue, 1 Jan 2019 15:21:12 -0500 Subject: [PATCH 1/2] .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 919486ec..af29c75f 100644 --- a/.gitignore +++ b/.gitignore @@ -195,3 +195,7 @@ gradle-app.setting # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties +# Eclipse and project files +.classpath +.settings +target \ No newline at end of file From 0788ff381309bf5759e14c4245b6d725c27263ab Mon Sep 17 00:00:00 2001 From: Joe Eugene Date: Tue, 1 Jan 2019 15:24:37 -0500 Subject: [PATCH 2/2] upgrade to aes-256 --- .gitignore | 1 + pom.xml | 477 ++++++++---------- .../platform/team/plugin/AmazonS3Wagon.java | 94 ++-- .../team/plugin/AmazonS3WagonTest.java | 94 ++-- 4 files changed, 305 insertions(+), 361 deletions(-) diff --git a/.gitignore b/.gitignore index af29c75f..e982045a 100644 --- a/.gitignore +++ b/.gitignore @@ -196,6 +196,7 @@ gradle-app.setting # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 # gradle/wrapper/gradle-wrapper.properties # Eclipse and project files +.project .classpath .settings target \ No newline at end of file diff --git a/pom.xml b/pom.xml index a8bd5f22..7e743c19 100644 --- a/pom.xml +++ b/pom.xml @@ -1,271 +1,210 @@ - - - 4.0.0 - - com.github.platform-team - aws-maven - jar - 6.0.0 - Amazon Web Services S3 Maven Wagon Support - Standard Maven wagon support for s3:// urls - - - 1.11.276 - 3.0.0 - - 1.7.25 - 1.2.3 - - 4.12 - 2.15.0 - - UTF-8 - - - - - ch.qos.logback - logback-classic - ${logback.version} - - - com.amazonaws - aws-java-sdk - ${amazonaws.version} - - - commons-logging - commons-logging - - - javax.mail - mail - - - stax - stax-api - - - stax - stax - - - - - org.apache.maven.wagon - wagon-provider-api - ${wagon.version} - provided - - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.apache.commons - commons-lang3 - 3.7 - - - - junit - junit - ${junit.version} - test - - - pl.pragmatists - JUnitParams - 1.1.1 - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - org.hamcrest - hamcrest-all - 1.3 - test - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.7.0 - - 1.8 - 1.8 - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - - org.apache.maven.plugins - maven-resources-plugin - 3.0.2 - - UTF-8 - - - - org.eluder.coveralls - coveralls-maven-plugin - 4.3.0 - - - org.jacoco - jacoco-maven-plugin - 0.8.0 - - - prepare-agent - - prepare-agent - - - - - - - - - - - - https://github.com/platform-team/aws-maven - - 2018 - - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 - repo - - - - - scm:git:git@github.com:platform-team/aws-maven.git - https://github.com/platform-team/aws-maven - - - - Platform Team - https://platform.team - - - - Travis CI - https://travis-ci.org/platform-team/aws-maven - - - - Github - https://github.com/platform-team/aws-maven/issues - - - - - moleksyuk - Mykhailo Oleksiuk - mykhailo.oleksiuk@gmail.com - - - - - + + + 4.0.0 + + com.github.platform-team + aws-maven + jar + 6.1.0 + Amazon Web Services S3 Maven Wagon Support + Standard Maven wagon support for s3:// urls + + + 1.11.276 + 3.0.0 + 1.7.25 + 1.2.3 + 4.12 + 2.15.0 + UTF-8 + + + + + ch.qos.logback + logback-classic + ${logback.version} + + + com.amazonaws + aws-java-sdk + ${amazonaws.version} + + + commons-logging + commons-logging + + + javax.mail + mail + + + stax + stax-api + + + stax + stax + + + + + org.apache.maven.wagon + wagon-provider-api + ${wagon.version} + provided + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.apache.commons + commons-lang3 + 3.7 + + + + junit + junit + ${junit.version} + test + + + pl.pragmatists + JUnitParams + 1.1.1 + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.hamcrest + hamcrest-all + 1.3 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.plugins + maven-resources-plugin + 3.0.2 + + UTF-8 + + + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + + org.jacoco + jacoco-maven-plugin + 0.8.0 + + + prepare-agent + + prepare-agent + + + + + + + + + + + + https://github.com/platform-team/aws-maven + + 2018 + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + repo + + + + + scm:git:git@github.com:platform-team/aws-maven.git + https://github.com/platform-team/aws-maven + + + + Platform Team + https://platform.team + + + + Travis CI + https://travis-ci.org/platform-team/aws-maven + + + + Github + https://github.com/platform-team/aws-maven/issues + + + + + moleksyuk + Mykhailo Oleksiuk + mykhailo.oleksiuk@gmail.com + + + + + \ No newline at end of file diff --git a/src/main/java/com/github/platform/team/plugin/AmazonS3Wagon.java b/src/main/java/com/github/platform/team/plugin/AmazonS3Wagon.java index 4f9e162a..31742be5 100644 --- a/src/main/java/com/github/platform/team/plugin/AmazonS3Wagon.java +++ b/src/main/java/com/github/platform/team/plugin/AmazonS3Wagon.java @@ -16,13 +16,31 @@ package com.github.platform.team.plugin; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.maven.wagon.ResourceDoesNotExistException; +import org.apache.maven.wagon.TransferFailedException; +import org.apache.maven.wagon.authentication.AuthenticationException; +import org.apache.maven.wagon.authentication.AuthenticationInfo; +import org.apache.maven.wagon.proxy.ProxyInfoProvider; +import org.apache.maven.wagon.repository.Repository; + import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.internal.Mimetypes; -import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -36,33 +54,17 @@ import com.github.platform.team.plugin.maven.AbstractWagon; import com.github.platform.team.plugin.util.IOUtils; import com.github.platform.team.plugin.util.S3Utils; -import org.apache.maven.wagon.ResourceDoesNotExistException; -import org.apache.maven.wagon.TransferFailedException; -import org.apache.maven.wagon.authentication.AuthenticationException; -import org.apache.maven.wagon.authentication.AuthenticationInfo; -import org.apache.maven.wagon.proxy.ProxyInfoProvider; -import org.apache.maven.wagon.repository.Repository; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** - * An implementation of the Maven Wagon interface that allows you to access the Amazon S3 service. URLs that reference - * the S3 service should be in the form of s3://bucket.name. As an example - * s3://static.springframework.org would put files into the static.springframework.org bucket - * on the S3 service. + * An implementation of the Maven Wagon interface that allows you to access the + * Amazon S3 service. URLs that reference the S3 service should be in the form + * of s3://bucket.name. As an example + * s3://static.springframework.org would put files into the + * static.springframework.org bucket on the S3 service. *

- * This implementation uses the username and passphrase portions of the server authentication - * metadata for credentials. + * This implementation uses the username and + * passphrase portions of the server authentication metadata for + * credentials. */ public final class AmazonS3Wagon extends AbstractWagon { @@ -90,7 +92,8 @@ public AmazonS3Wagon() { this.baseDirectory = baseDirectory; } - private static ObjectMetadata getObjectMetadata(AmazonS3 amazonS3, String bucketName, String baseDirectory, String resourceName) { + private static ObjectMetadata getObjectMetadata(AmazonS3 amazonS3, String bucketName, String baseDirectory, + String resourceName) { return amazonS3.getObjectMetadata(bucketName, getKey(baseDirectory, resourceName)); } @@ -120,7 +123,8 @@ private static String getResourceName(String key, Pattern pattern) { return key; } - private static void mkdirs(AmazonS3 amazonS3, String bucketName, String path, int index) throws TransferFailedException { + private static void mkdirs(AmazonS3 amazonS3, String bucketName, String path, int index) + throws TransferFailedException { int directoryIndex = path.indexOf('/', index) + 1; if (directoryIndex != 0) { @@ -142,35 +146,32 @@ private static PutObjectRequest createDirectoryPutObjectRequest(String bucketNam ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(0); + objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); - return new PutObjectRequest(bucketName, key, inputStream, objectMetadata).withCannedAcl(CannedAccessControlList.PublicRead); + return new PutObjectRequest(bucketName, key, inputStream, objectMetadata); } - private static String getBucketRegion(AWSCredentialsProvider credentialsProvider, ClientConfiguration clientConfiguration, String bucketName) { - return AmazonS3Client.builder() - .withCredentials(credentialsProvider) - .withClientConfiguration(clientConfiguration) - .enableForceGlobalBucketAccess() - .build() + private static String getBucketRegion(AWSCredentialsProvider credentialsProvider, + ClientConfiguration clientConfiguration, String bucketName) { + return AmazonS3Client.builder().withCredentials(credentialsProvider) + .withClientConfiguration(clientConfiguration).enableForceGlobalBucketAccess().build() .getBucketLocation(bucketName); } @Override protected void connectToRepository(Repository repository, AuthenticationInfo authenticationInfo, - ProxyInfoProvider proxyInfoProvider) throws AuthenticationException { + ProxyInfoProvider proxyInfoProvider) throws AuthenticationException { if (this.amazonS3 == null) { - AWSMavenCredentialsProviderChain credentialsProvider = - new AWSMavenCredentialsProviderChain(authenticationInfo); + AWSMavenCredentialsProviderChain credentialsProvider = new AWSMavenCredentialsProviderChain( + authenticationInfo); ClientConfiguration clientConfiguration = S3Utils.getClientConfiguration(proxyInfoProvider); this.bucketName = S3Utils.getBucketName(repository); this.baseDirectory = S3Utils.getBaseDirectory(repository); - this.amazonS3 = AmazonS3Client.builder() - .withCredentials(credentialsProvider) + this.amazonS3 = AmazonS3Client.builder().withCredentials(credentialsProvider) .withClientConfiguration(clientConfiguration) - .withRegion(getBucketRegion(credentialsProvider, clientConfiguration, this.bucketName)) - .build(); + .withRegion(getBucketRegion(credentialsProvider, clientConfiguration, this.bucketName)).build(); } } @@ -194,7 +195,8 @@ protected boolean doesRemoteResourceExist(String resourceName) { @Override protected boolean isRemoteResourceNewer(String resourceName, long timestamp) throws ResourceDoesNotExistException { try { - Date lastModified = getObjectMetadata(this.amazonS3, this.bucketName, this.baseDirectory, resourceName).getLastModified(); + Date lastModified = getObjectMetadata(this.amazonS3, this.bucketName, this.baseDirectory, resourceName) + .getLastModified(); return lastModified == null || lastModified.getTime() > timestamp; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", resourceName), e); @@ -247,15 +249,16 @@ protected void getResource(String resourceName, File destination, TransferProgre } catch (FileNotFoundException e) { throw new TransferFailedException(String.format("Cannot write file to '%s'", destination), e); } catch (IOException e) { - throw new TransferFailedException(String.format("Cannot read from '%s' and write to '%s'", resourceName, destination), e); + throw new TransferFailedException( + String.format("Cannot read from '%s' and write to '%s'", resourceName, destination), e); } finally { IOUtils.closeQuietly(in, out); } } @Override - protected void putResource(File source, String destination, TransferProgress transferProgress) throws TransferFailedException, - ResourceDoesNotExistException { + protected void putResource(File source, String destination, TransferProgress transferProgress) + throws TransferFailedException, ResourceDoesNotExistException { String key = getKey(this.baseDirectory, destination); mkdirs(amazonS3, this.bucketName, key, 0); @@ -265,6 +268,7 @@ protected void putResource(File source, String destination, TransferProgress tra ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(source.length()); objectMetadata.setContentType(Mimetypes.getInstance().getMimetype(source)); + objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); in = new TransferProgressFileInputStream(source, transferProgress); diff --git a/src/test/java/com/github/platform/team/plugin/AmazonS3WagonTest.java b/src/test/java/com/github/platform/team/plugin/AmazonS3WagonTest.java index c1211456..3d949637 100644 --- a/src/test/java/com/github/platform/team/plugin/AmazonS3WagonTest.java +++ b/src/test/java/com/github/platform/team/plugin/AmazonS3WagonTest.java @@ -16,9 +16,36 @@ package com.github.platform.team.plugin; +import static com.github.platform.team.plugin.maven.matchers.Matchers.eq; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import org.apache.maven.wagon.ResourceDoesNotExistException; +import org.apache.maven.wagon.TransferFailedException; +import org.apache.maven.wagon.WagonException; +import org.apache.maven.wagon.authentication.AuthenticationInfo; +import org.apache.maven.wagon.repository.Repository; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -28,33 +55,6 @@ import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.github.platform.team.plugin.data.TransferProgress; -import org.apache.maven.wagon.ResourceDoesNotExistException; -import org.apache.maven.wagon.TransferFailedException; -import org.apache.maven.wagon.WagonException; -import org.apache.maven.wagon.authentication.AuthenticationInfo; -import org.apache.maven.wagon.repository.Repository; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.ArgumentCaptor; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; - -import static com.github.platform.team.plugin.maven.matchers.Matchers.eq; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; public class AmazonS3WagonTest { @@ -76,8 +76,7 @@ public class AmazonS3WagonTest { private final TransferProgress transferProgress = mock(TransferProgress.class); - private final AmazonS3Wagon wagon = - new AmazonS3Wagon(this.amazonS3, BUCKET_NAME, BASE_DIRECTORY); + private final AmazonS3Wagon wagon = new AmazonS3Wagon(this.amazonS3, BUCKET_NAME, BASE_DIRECTORY); // Run only locally against own aws keys @Ignore @@ -121,23 +120,23 @@ private List getBuckets() { @Test public void doesRemoteResourceExistExists() { - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenReturn(this.objectMetadata); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenReturn(this.objectMetadata); assertTrue(this.wagon.doesRemoteResourceExist(FILE_NAME)); } @Test public void doesRemoteResourceExistDoesNotExist() { - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenThrow(new AmazonServiceException("")); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenThrow(new AmazonServiceException("")); assertFalse(this.wagon.doesRemoteResourceExist(FILE_NAME)); } @Test public void isRemoteResourceNewerNewer() throws ResourceDoesNotExistException { when(this.objectMetadata.getLastModified()).thenReturn(new Date()); - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenReturn(this.objectMetadata); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenReturn(this.objectMetadata); assertTrue(this.wagon.isRemoteResourceNewer(FILE_NAME, 0)); } @@ -145,24 +144,24 @@ public void isRemoteResourceNewerNewer() throws ResourceDoesNotExistException { @Test public void isRemoteResourceNewerOlder() throws ResourceDoesNotExistException { when(this.objectMetadata.getLastModified()).thenReturn(new Date()); - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenReturn(this.objectMetadata); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenReturn(this.objectMetadata); assertFalse(this.wagon.isRemoteResourceNewer(FILE_NAME, Long.MAX_VALUE)); } @Test public void isRemoteResourceNewerNoLastModified() throws ResourceDoesNotExistException { - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenReturn(this.objectMetadata); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenReturn(this.objectMetadata); assertTrue(this.wagon.isRemoteResourceNewer(FILE_NAME, 0)); } @Test(expected = ResourceDoesNotExistException.class) public void isRemoteResourceNewerDoesNotExist() throws ResourceDoesNotExistException { - when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenThrow(new AmazonServiceException("")); + when(this.amazonS3.getObjectMetadata(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenThrow(new AmazonServiceException("")); this.wagon.isRemoteResourceNewer(FILE_NAME, 0); } @@ -215,8 +214,8 @@ public void listDirectoryDoesNotExist() throws ResourceDoesNotExistException { @Test public void getResource() throws TransferFailedException, FileNotFoundException, ResourceDoesNotExistException { - when(this.amazonS3.getObject(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenReturn(this.s3Object); + when(this.amazonS3.getObject(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenReturn(this.s3Object); when(this.s3Object.getObjectContent()) .thenReturn(new S3ObjectInputStream(new FileInputStream("src/test/resources/test.txt"), null)); @@ -231,8 +230,8 @@ public void getResource() throws TransferFailedException, FileNotFoundException, @Test(expected = ResourceDoesNotExistException.class) public void getResourceSourceDoesNotExist() throws TransferFailedException, ResourceDoesNotExistException { - when(this.amazonS3.getObject(AmazonS3WagonTest.BUCKET_NAME, - BASE_DIRECTORY + FILE_NAME)).thenThrow(new AmazonServiceException("")); + when(this.amazonS3.getObject(AmazonS3WagonTest.BUCKET_NAME, BASE_DIRECTORY + FILE_NAME)) + .thenThrow(new AmazonServiceException("")); File target = new File("target/robots.txt"); this.wagon.getResource(FILE_NAME, target, this.transferProgress); } @@ -250,7 +249,8 @@ public void putResource() throws TransferFailedException, ResourceDoesNotExistEx assertEquals(BUCKET_NAME, putObjectRequests.get(i).getBucketName()); assertNotNull(putObjectRequests.get(i).getInputStream()); assertEquals(0, putObjectRequests.get(i).getMetadata().getContentLength()); - assertEquals(CannedAccessControlList.PublicRead, putObjectRequests.get(i).getCannedAcl()); + // remove assertEquals(CannedAccessControlList.PublicRead, + // putObjectRequests.get(i).getCannedAcl()); } assertEquals("foo/", putObjectRequests.get(0).getKey());