Pivotal ID # 184975504: S3 submission resubmission

buildSrc/src/main/kotlin/Dependencies.kt

@@ -8,6 +8,7 @@ import TestVersions.JunitExtensionsVersion
 import TestVersions.JunitVersion
 import TestVersions.MockKVersion
 import TestVersions.RabbitmqMockVersion
+import TestVersions.TestContainerS3mockVersion
 import TestVersions.TestContainerVersion
 import TestVersions.WiremockVersion
 import TestVersions.XmlUnitVersion
@@ -37,6 +38,7 @@ import Versions.OkHttpLoggingVersion
 import Versions.PoiVersion
 import Versions.Retrofit2Version
 import Versions.RxJava2Version
+import Versions.S3Version
 import Versions.ServletVersion
 import Versions.SpringAdminVersion
 import Versions.SpringVersion
@@ -59,6 +61,7 @@ object TestVersions {
     const val WiremockVersion = "2.27.2"
     const val RabbitmqMockVersion = "1.1.0"
     const val TestContainerVersion = "1.16.2"
+    const val TestContainerS3mockVersion = "2.11.0"
     const val AwaitilityVersion = "4.2.0"
 }
 
@@ -88,6 +91,7 @@ object Versions {
     const val JwtVersion = "0.9.1"
     const val H2Version = "1.4.197"
     const val ServletVersion = "4.0.1"
+    const val S3Version = "1.12.293"
     const val HibernateEMVersion = "5.3.5.Final"
     const val JschVersion = "0.1.55"
     const val Retrofit2Version = "2.9.0"
@@ -134,6 +138,7 @@ object TestDependencies {
 
     // Test Containers
     const val TestContainerMysql = "org.testcontainers:mysql:$TestContainerVersion"
+    const val TestContainerS3mock = "com.adobe.testing:s3mock-testcontainers:$TestContainerS3mockVersion"
     const val TestContainerMongoDb = "org.testcontainers:mongodb:$TestContainerVersion"
     const val TestContainer = "org.testcontainers:testcontainers:$TestContainerVersion"
     const val TestContainerJUnit = "org.testcontainers:junit-jupiter:$TestContainerVersion"
@@ -162,6 +167,7 @@ object Dependencies {
     const val MongockSpringDataV3 = "com.github.cloudyrock.mongock:mongodb-springdata-v3-driver:$MongockVersion"
 
     // Misc
+    const val AwsS3 = "com.amazonaws:aws-java-sdk-s3:$S3Version"
     const val ServletApi = "javax.servlet:javax.servlet-api:$ServletVersion"
     const val Logback = "ch.qos.logback:logback-classic:$LogbackVersion"
     const val Jwt = "io.jsonwebtoken:jjwt:$JwtVersion"

client/fire-webclient/build.gradle.kts

@@ -1,3 +1,4 @@
+import Dependencies.AwsS3
 import Dependencies.JSONOrg
 import Dependencies.KotlinLogging
 import Dependencies.KotlinReflect
@@ -32,6 +33,7 @@ dependencies {
     implementation(JSONOrg)
     implementation(SpringWeb)
     implementation(SpringRetry)
+    implementation(AwsS3)
 
     testApi(project(CommonsTest))
     testApi(project(JsonLibrary))

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/api/S3Client.kt

@@ -0,0 +1,30 @@
+package uk.ac.ebi.fire.client.api
+
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.GetObjectRequest
+import com.amazonaws.services.s3.model.S3Object
+import uk.ac.ebi.fire.client.integration.web.FireS3Client
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+
+class S3Client(
+    private val amazonS3Client: AmazonS3,
+    private val bucketName: String,
+) : FireS3Client {
+    override fun downloadByPath(path: String): File? {
+        val stream = getFireObjectByPath(path).objectContent
+        val tmp = File.createTempFile(DEFAULT_PREFIX, path.substringAfterLast("/"))
+        Files.copy(stream, tmp.toPath(), StandardCopyOption.REPLACE_EXISTING)
+        return tmp
+    }
+
+    private fun getFireObjectByPath(path: String?): S3Object {
+        val getObjectRequest = GetObjectRequest(bucketName, path)
+        return amazonS3Client.getObject(getObjectRequest)
+    }
+
+    companion object {
+        const val DEFAULT_PREFIX = "tmp_s3_file"
+    }
+}

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/integration/web/FireClient.kt

@@ -3,16 +3,16 @@ package uk.ac.ebi.fire.client.integration.web
 import uk.ac.ebi.fire.client.model.FireApiFile
 import java.io.File
 
+interface FireClient : FireWebClient, FireS3Client
+
 @Suppress("TooManyFunctions")
-interface FireClient {
+interface FireWebClient {
     fun save(file: File, md5: String, size: Long): FireApiFile
 
     fun setPath(fireOid: String, path: String)
 
     fun unsetPath(fireOid: String)
 
-    fun downloadByPath(path: String): File?
-
     fun findByMd5(md5: String): List<FireApiFile>
 
     fun findByPath(path: String): FireApiFile?
@@ -25,3 +25,7 @@ interface FireClient {
 
     fun delete(fireOid: String)
 }
+
+interface FireS3Client {
+    fun downloadByPath(path: String): File?
+}

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/integration/web/FireClientFactory.kt

@@ -1,5 +1,10 @@
 package uk.ac.ebi.fire.client.integration.web
 
+import com.amazonaws.auth.AWSStaticCredentialsProvider
+import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.client.builder.AwsClientBuilder
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.AmazonS3Client
 import org.springframework.http.client.SimpleClientHttpRequestFactory
 import org.springframework.retry.support.RetryTemplate
 import org.springframework.retry.support.RetryTemplateBuilder
@@ -8,26 +13,43 @@ import org.springframework.web.client.ResourceAccessException
 import org.springframework.web.client.RestTemplate
 import org.springframework.web.util.DefaultUriBuilderFactory
 import uk.ac.ebi.fire.client.api.FireWebClient
+import uk.ac.ebi.fire.client.api.S3Client
 
 private const val FIRE_API_BASE = "fire"
 
 class FireClientFactory private constructor() {
     companion object {
-        fun create(tmpDirPath: String, config: FireConfig): FireClient {
-            val restTemplate = createRestTemplate(config.fireHost, config.fireVersion, config.username, config.password)
-            return FireWebClient(tmpDirPath, restTemplate)
-        }
-
         fun create(
             tmpDirPath: String,
             fireConfig: FireConfig,
+            s3Config: S3Config,
             retryConfig: RetryConfig,
         ): FireClient =
             RetryWebClient(
-                create(tmpDirPath, fireConfig),
+                createHttpClient(tmpDirPath, fireConfig),
+                createS3Client(s3Config),
                 createRetryTemplate(retryConfig)
             )
 
+        fun amazonS3Client(s3Config: S3Config): AmazonS3 {
+            val basicAWSCredentials = BasicAWSCredentials(s3Config.accessKey, s3Config.secretKey)
+            val endpointConfiguration = AwsClientBuilder.EndpointConfiguration(s3Config.endpoint, s3Config.region)
+            return AmazonS3Client.builder()
+                .withEndpointConfiguration(endpointConfiguration)
+                .withPathStyleAccessEnabled(true)
+                .withCredentials(AWSStaticCredentialsProvider(basicAWSCredentials))
+                .build()
+        }
+
+        private fun createS3Client(s3Config: S3Config): FireS3Client {
+            return S3Client(amazonS3Client(s3Config), s3Config.bucket)
+        }
+
+        private fun createHttpClient(tmpDirPath: String, config: FireConfig): FireWebClient {
+            val restTemplate = createRestTemplate(config.fireHost, config.fireVersion, config.username, config.password)
+            return FireWebClient(tmpDirPath, restTemplate)
+        }
+
         private fun createRetryTemplate(config: RetryConfig): RetryTemplate = RetryTemplateBuilder()
             .exponentialBackoff(config.initialInterval, config.multiplier, config.maxInterval)
             .retryOn(listOf(HttpServerErrorException::class.java, ResourceAccessException::class.java))
@@ -47,6 +69,14 @@ class FireClientFactory private constructor() {
     }
 }
 
+data class S3Config(
+    val accessKey: String,
+    val secretKey: String,
+    val region: String,
+    val endpoint: String,
+    val bucket: String,
+)
+
 data class FireConfig(
     val fireHost: String,
     val fireVersion: String,

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/integration/web/RetryWebClient.kt

@@ -7,7 +7,8 @@ import java.io.File
 
 @Suppress("TooManyFunctions")
 internal class RetryWebClient(
-    private val fireClient: FireClient,
+    private val fireClient: FireWebClient,
+    private val fireS3Client: FireS3Client,
     private val template: RetryTemplate,
 ) : FireClient {
     override fun save(file: File, md5: String, size: Long): FireApiFile {
@@ -25,11 +26,6 @@ internal class RetryWebClient(
         template.execute(opt) { fireClient.unsetPath(fireOid) }
     }
 
-    override fun downloadByPath(path: String): File? {
-        val opt = "Download file path='$path'"
-        return template.execute(opt) { fireClient.downloadByPath(path) }
-    }
-
     override fun findByMd5(md5: String): List<FireApiFile> {
         val opt = "Find file md5='$md5'"
         return template.execute(opt) { fireClient.findByMd5(md5) }
@@ -59,4 +55,9 @@ internal class RetryWebClient(
         val opt = "Delete file fireOid='$fireOid'"
         template.execute(opt) { fireClient.delete(fireOid) }
     }
+
+    override fun downloadByPath(path: String): File? {
+        val opt = "Dowload file path='$path'"
+        return template.execute(opt) { fireS3Client.downloadByPath(path) }
+    }
 }

submission/submission-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/ApplicationProperties.kt

@@ -35,6 +35,12 @@ data class FireProperties(
     val username: String,
     val password: String,
     val retry: RetryProperties,
+
+    val s3AccessKey: String,
+    val s3SecretKey: String,
+    val s3region: String,
+    val s3endpoint: String,
+    val s3bucket: String,
 )
 
 data class ValidatorProperties(

submission/submission-webapp/build.gradle.kts

@@ -43,6 +43,7 @@ import TestDependencies.TestContainer
 import TestDependencies.TestContainerJUnit
 import TestDependencies.TestContainerMongoDb
 import TestDependencies.TestContainerMysql
+import TestDependencies.TestContainerS3mock
 import TestDependencies.Wiremock
 import TestDependencies.XmlUnitCore
 import TestDependencies.XmlUnitMatchers
@@ -127,6 +128,7 @@ dependencies {
     testImplementation(XmlUnitMatchers)
 
     testImplementation(TestContainerMysql)
+    testImplementation(TestContainerS3mock)
     testImplementation(TestContainerMongoDb)
     testImplementation(TestContainer)
     testImplementation(TestContainerJUnit)

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/common/TestConfig.kt

@@ -1,9 +1,13 @@
 package ac.uk.ebi.biostd.itest.common
 
+import ac.uk.ebi.biostd.common.properties.ApplicationProperties
 import ac.uk.ebi.biostd.persistence.repositories.UserDataRepository
+import com.amazonaws.services.s3.AmazonS3
 import ebi.ac.uk.security.service.SecurityService
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import uk.ac.ebi.fire.client.integration.web.FireClientFactory
+import uk.ac.ebi.fire.client.integration.web.S3Config
 
 @Configuration
 class TestConfig {
@@ -18,4 +22,18 @@ class TestConfig {
 
     @Bean(name = ["FailCollectionValidator"])
     fun failCollectionValidator(): FailCollectionValidator = FailCollectionValidator()
+
+    @Bean
+    fun s3Service(appProperties: ApplicationProperties): AmazonS3 {
+        val fireProps = appProperties.fire
+        return FireClientFactory.amazonS3Client(
+            S3Config(
+                accessKey = fireProps.s3AccessKey,
+                secretKey = fireProps.s3SecretKey,
+                region = fireProps.s3region,
+                endpoint = fireProps.s3endpoint,
+                bucket = fireProps.s3bucket
+            )
+        )
+    }
 }

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/itest/ITestListener.kt

@@ -7,6 +7,7 @@ import ac.uk.ebi.biostd.itest.common.FIRE_USERNAME
 import ac.uk.ebi.biostd.itest.common.SpecificMySQLContainer
 import ac.uk.ebi.biostd.itest.wiremock.TestWireMockTransformer
 import ac.uk.ebi.biostd.itest.wiremock.TestWireMockTransformer.Companion.newTransformer
+import com.adobe.testing.s3mock.testcontainers.S3MockContainer
 import com.github.tomakehurst.wiremock.WireMockServer
 import com.github.tomakehurst.wiremock.client.WireMock
 import com.github.tomakehurst.wiremock.client.WireMock.post
@@ -59,10 +60,16 @@ class ITestListener : TestExecutionListener {
                 .willReturn(WireMock.aResponse().withTransformers(TestWireMockTransformer.name))
         )
         fireApiMock.start()
-
         System.setProperty("app.fire.host", fireApiMock.baseUrl())
         System.setProperty("app.fire.username", FIRE_USERNAME)
         System.setProperty("app.fire.password", FIRE_PASSWORD)
+
+        s3Container.start()
+        System.setProperty("app.fire.s3AccessKey", "anyKey")
+        System.setProperty("app.fire.s3SecretKey", "anySecret")
+        System.setProperty("app.fire.s3region", "us-east-1")
+        System.setProperty("app.fire.s3endpoint", s3Container.httpEndpoint)
+        System.setProperty("app.fire.s3bucket", defaultBucket)
     }
 
     private fun appPropertiesSetup() {
@@ -78,6 +85,8 @@ class ITestListener : TestExecutionListener {
 
     companion object {
         val testAppFolder = Files.createTempDirectory("test-app-folder").toFile()
+        val defaultBucket = "bio-fire-bucket"
+
         internal val nfsSubmissionPath = testAppFolder.createDirectory("submission")
         internal val fireSubmissionPath = testAppFolder.createDirectory("submission-fire")
         internal val firePath = testAppFolder.createDirectory("fire-db")
@@ -92,9 +101,10 @@ class ITestListener : TestExecutionListener {
         internal val magicDirPath = testAppFolder.createDirectory("magic")
         internal val dropboxPath = testAppFolder.createDirectory("dropbox")
 
-        internal val fireApiMock = createFireApiMock(fireFtpPath)
+        private val fireApiMock = createFireApiMock(fireFtpPath)
         private val mongoContainer = createMongoContainer()
         private val mysqlContainer = createMysqlContainer()
+        private val s3Container = createMockS3Container()
 
         val enableFire get() = System.getProperty("enableFire").toBoolean()
         val storageMode get() = if (enableFire) StorageMode.FIRE else StorageMode.NFS
@@ -111,17 +121,20 @@ class ITestListener : TestExecutionListener {
                 .withInitScript(MYSQL_SCHEMA)
                 .withStartupCheckStrategy(MinimumDurationRunningStartupCheckStrategy(ofSeconds(MINIMUM_RUNNING_TIME)))
 
+        private fun createMockS3Container(): S3MockContainer {
+            return S3MockContainer("latest")
+                .withInitialBuckets(defaultBucket)
+        }
+
         private fun createFireApiMock(ftpDir: File): WireMockServer {
             val factor = System.getenv("ITEST_FAIL_FACTOR")?.toInt()
             val transformer = newTransformer(fireSubmissionPath.toPath(), ftpDir.toPath(), firePath.toPath(), factor)
-
             return WireMockServer(WireMockConfiguration().dynamicPort().extensions(transformer))
         }
 
         private fun File.createDirectory(path: String): File {
             val file = resolve(path)
             file.mkdir()
-
             return file
         }
     }