Pivotal ID #185392780: User FTP files API support (#724)

https://www.pivotaltracker.com/story/show/185392780
- added ftp magic folder support

infrastructure/src/main/resources/setup/mysql/Schema.sql

@@ -52,6 +52,7 @@ CREATE TABLE User
     fullName             VARCHAR(255) NULL,
     keyTime              BIGINT NOT NULL,
     login                VARCHAR(255) NULL,
+    storageMode          VARCHAR(255) NOT NULL,
     passwordDigest       LONGBLOB NULL,
     secret               VARCHAR(255) NULL,
     superuser            BIT    NOT NULL,

submission/persistence-sql/src/main/kotlin/ac/uk/ebi/biostd/persistence/model/DbUser.kt

@@ -1,10 +1,13 @@
 package ac.uk.ebi.biostd.persistence.model
 
+import ac.uk.ebi.biostd.common.properties.StorageMode
 import java.time.OffsetDateTime
 import javax.persistence.CascadeType.MERGE
 import javax.persistence.CascadeType.PERSIST
 import javax.persistence.Column
 import javax.persistence.Entity
+import javax.persistence.EnumType
+import javax.persistence.Enumerated
 import javax.persistence.FetchType
 import javax.persistence.GeneratedValue
 import javax.persistence.Id
@@ -76,6 +79,10 @@ class DbUser(
     @Column
     var notificationsEnabled: Boolean = false,
 
+    @Enumerated(EnumType.STRING)
+    @Column(name = "storageMode")
+    var storageMode: StorageMode,
+
     @Column
     var orcid: String? = null,
 ) {

submission/submission-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/SecurityProperties.kt

@@ -6,11 +6,22 @@ data class SecurityProperties(
     val captchaKey: String = "",
     val checkCaptcha: Boolean = false,
     val tokenHash: String,
-    val filesDirPath: String,
-    val magicDirPath: String,
     val environment: String,
     val requireActivation: Boolean = false,
-    @NestedConfigurationProperty val instanceKeys: InstanceKeys = InstanceKeys()
+
+    @NestedConfigurationProperty
+    val instanceKeys: InstanceKeys = InstanceKeys(),
+    val filesProperties: FilesProperties,
+)
+
+enum class StorageMode {
+    FTP, NFS
+}
+
+data class FilesProperties(
+    val defaultMode: StorageMode,
+    val filesDirPath: String,
+    val magicDirPath: String,
 )
 
 data class InstanceKeys(

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/integration/SecurityModuleConfig.kt

@@ -35,18 +35,25 @@ class SecurityModuleConfig(
     private val queryService: SubmissionMetaQueryService,
     private val userPermissionsService: UserPermissionsService,
     private val eventsPublisherService: EventsPublisherService,
-    private var props: SecurityProperties
+    private var props: SecurityProperties,
 ) {
     fun securityService(): ISecurityService = securityService
     fun securityQueryService(): ISecurityQueryService = securityQueryService
     fun groupService(): IGroupService = groupService
     fun securityFilter(): ISecurityFilter = securityFilter
     fun userPrivilegesService(): IUserPrivilegesService = userPrivilegesService
 
-    private val groupService by lazy { GroupService(groupRepository, userRepo, props.filesDirPath) }
-    private val securityQueryService by lazy { SecurityQueryService(securityUtil, profileService, userRepo) }
+    private val groupService by lazy { GroupService(groupRepository, userRepo, props.filesProperties.filesDirPath) }
+    private val securityQueryService by lazy { SecurityQueryService(securityUtil, profileService, userRepo, props) }
     private val securityService by lazy {
-        SecurityService(userRepo, securityUtil, props, profileService, captchaVerifier, eventsPublisherService)
+        SecurityService(
+            userRepo,
+            securityUtil,
+            props,
+            profileService,
+            captchaVerifier,
+            eventsPublisherService
+        )
     }
 
     private val securityFilter by lazy { SecurityFilter(props.environment, securityQueryService) }
@@ -57,7 +64,7 @@ class SecurityModuleConfig(
     private val captchaVerifier by lazy { CaptchaVerifier(RestTemplate(), props) }
     private val objectMapper by lazy { JacksonFactory.createMapper() }
     private val jwtParser by lazy { Jwts.parser()!! }
-    private val profileService by lazy { ProfileService(Paths.get(props.filesDirPath)) }
+    private val profileService by lazy { ProfileService(Paths.get(props.filesProperties.filesDirPath)) }
     private val securityUtil by lazy {
         SecurityUtil(jwtParser, objectMapper, tokenRepo, userRepo, props.tokenHash, props.instanceKeys)
     }

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/integration/model/api/SecurityUser.kt

@@ -12,8 +12,8 @@ data class SecurityUser(
     val orcid: String?,
     val secret: String,
     val superuser: Boolean,
-    val magicFolder: MagicFolder,
-    val groupsFolders: List<GroupMagicFolder>,
+    val userFolder: UserFolder,
+    val groupsFolders: List<GroupFolder>,
     val permissions: Set<SecurityPermission>,
     val notificationsEnabled: Boolean,
 ) {
@@ -22,8 +22,13 @@ data class SecurityUser(
 
 data class SecurityPermission(val accessType: AccessType, val accessTag: String)
 
-data class MagicFolder(val relativePath: Path, val path: Path) {
-    fun resolve(subPath: String): Path = path.resolve(subPath)
+sealed interface UserFolder {
+    val relativePath: Path
 }
 
-data class GroupMagicFolder(val groupName: String, val path: Path, val description: String? = null)
+data class FtpUserFolder(override val relativePath: Path) : UserFolder
+data class NfsUserFolder(override val relativePath: Path, val path: Path) : UserFolder
+
+fun NfsUserFolder.resolve(subPath: String): Path = path.resolve(subPath)
+
+data class GroupFolder(val groupName: String, val path: Path, val description: String? = null)

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/service/ProfileService.kt

@@ -1,12 +1,15 @@
 package ebi.ac.uk.security.service
 
+import ac.uk.ebi.biostd.common.properties.StorageMode
 import ac.uk.ebi.biostd.persistence.model.DbAccessPermission
 import ac.uk.ebi.biostd.persistence.model.DbUser
 import ac.uk.ebi.biostd.persistence.model.DbUserGroup
-import ebi.ac.uk.security.integration.model.api.GroupMagicFolder
-import ebi.ac.uk.security.integration.model.api.MagicFolder
+import ebi.ac.uk.security.integration.model.api.FtpUserFolder
+import ebi.ac.uk.security.integration.model.api.GroupFolder
+import ebi.ac.uk.security.integration.model.api.NfsUserFolder
 import ebi.ac.uk.security.integration.model.api.SecurityPermission
 import ebi.ac.uk.security.integration.model.api.SecurityUser
+import ebi.ac.uk.security.integration.model.api.UserFolder
 import ebi.ac.uk.security.integration.model.api.UserInfo
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -22,7 +25,7 @@ class ProfileService(private val filesDirPath: Path) {
         orcid = user.orcid,
         secret = user.secret,
         superuser = user.superuser,
-        magicFolder = userMagicFolder(user.secret, user.id),
+        userFolder = userMagicFolder(user.storageMode, user.secret, user.id),
         groupsFolders = groupsMagicFolder(user.groups),
         permissions = getPermissions(user.permissions),
         notificationsEnabled = user.notificationsEnabled
@@ -31,14 +34,26 @@ class ProfileService(private val filesDirPath: Path) {
     private fun getPermissions(permissions: Set<DbAccessPermission>): Set<SecurityPermission> =
         permissions.mapTo(mutableSetOf()) { SecurityPermission(it.accessType, it.accessTag.name) }
 
-    private fun groupsMagicFolder(groups: Set<DbUserGroup>): List<GroupMagicFolder> =
-        groups.map { GroupMagicFolder(it.name, groupMagicFolder(it), it.description) }
+    private fun groupsMagicFolder(groups: Set<DbUserGroup>): List<GroupFolder> =
+        groups.map { GroupFolder(it.name, groupMagicFolder(it), it.description) }
 
     private fun groupMagicFolder(it: DbUserGroup) = Paths.get("$filesDirPath/${magicPath(it.secret, it.id, "b")}")
 
-    private fun userMagicFolder(secret: String, id: Long): MagicFolder {
-        val relativePath = magicPath(secret, id, "a")
-        return MagicFolder(Paths.get(relativePath), Paths.get("$filesDirPath/$relativePath"))
+    private fun userMagicFolder(folderType: StorageMode, secret: String, id: Long): UserFolder {
+        fun nfsFolder(): NfsUserFolder {
+            val relativePath = magicPath(secret, id, "a")
+            return NfsUserFolder(Paths.get(relativePath), Paths.get("$filesDirPath/$relativePath"))
+        }
+
+        fun ftpFolder(): FtpUserFolder {
+            val relativePath = magicPath(secret, id, "a")
+            return FtpUserFolder(Paths.get(relativePath))
+        }
+
+        return when (folderType) {
+            StorageMode.FTP -> ftpFolder()
+            StorageMode.NFS -> nfsFolder()
+        }
     }
 
     private fun magicPath(secret: String, id: Long, suffix: String) = "${secret.take(2)}/${secret.drop(2)}-$suffix$id"

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/service/SecurityQueryService.kt

@@ -1,5 +1,6 @@
 package ebi.ac.uk.security.service
 
+import ac.uk.ebi.biostd.common.properties.SecurityProperties
 import ac.uk.ebi.biostd.persistence.model.DbUser
 import ac.uk.ebi.biostd.persistence.repositories.UserDataRepository
 import ebi.ac.uk.security.integration.components.ISecurityQueryService
@@ -13,6 +14,7 @@ class SecurityQueryService(
     private val securityUtil: SecurityUtil,
     private val profileService: ProfileService,
     private val userRepository: UserDataRepository,
+    private val securityProperties: SecurityProperties,
 ) : ISecurityQueryService {
     override fun existsByEmail(email: String, onlyActive: Boolean): Boolean {
         return if (onlyActive) userRepository.existsByEmailAndActive(email, true)
@@ -41,6 +43,7 @@ class SecurityQueryService(
             email = email,
             fullName = username,
             secret = securityUtil.newKey(),
+            storageMode = securityProperties.filesProperties.defaultMode,
             passwordDigest = ByteArray(0),
             notificationsEnabled = false
         )

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/service/SecurityService.kt

@@ -23,6 +23,8 @@ import ebi.ac.uk.security.integration.exception.LoginException
 import ebi.ac.uk.security.integration.exception.UserAlreadyRegister
 import ebi.ac.uk.security.integration.exception.UserNotFoundByEmailException
 import ebi.ac.uk.security.integration.exception.UserPendingRegistrationException
+import ebi.ac.uk.security.integration.model.api.FtpUserFolder
+import ebi.ac.uk.security.integration.model.api.NfsUserFolder
 import ebi.ac.uk.security.integration.model.api.SecurityUser
 import ebi.ac.uk.security.integration.model.api.UserInfo
 import ebi.ac.uk.security.persistence.getActiveByEmail
@@ -152,15 +154,26 @@ open class SecurityService(
         val dbUser = userRepository.save(toActivate.apply { activationKey = null; active = true })
         val securityUser = profileService.asSecurityUser(dbUser)
 
-        FileUtils.getOrCreateFolder(securityUser.magicFolder.path.parent, RWX__X___)
-        FileUtils.getOrCreateFolder(securityUser.magicFolder.path, RWXRWX___)
-        FileUtils.createSymbolicLink(symLinkPath(securityUser.email), securityUser.magicFolder.path, RWXRWX___)
+        createMagicFolder(securityUser)
         return securityUser
     }
 
+    private fun createMagicFolder(user: SecurityUser) {
+        when (user.userFolder) {
+            is FtpUserFolder -> TODO()
+            is NfsUserFolder -> createNfsMagicFolder(user.email, user.userFolder)
+        }
+    }
+
+    private fun createNfsMagicFolder(email: String, magicFolder: NfsUserFolder) {
+        FileUtils.getOrCreateFolder(magicFolder.path.parent, RWX__X___)
+        FileUtils.getOrCreateFolder(magicFolder.path, RWXRWX___)
+        FileUtils.createSymbolicLink(symLinkPath(email), magicFolder.path, RWXRWX___)
+    }
+
     private fun symLinkPath(userEmail: String): Path {
         val prefixFolder = userEmail.substring(0, 1).lowercase()
-        return Paths.get("${securityProps.magicDirPath}/$prefixFolder/$userEmail")
+        return Paths.get("${securityProps.filesProperties.magicDirPath}/$prefixFolder/$userEmail")
     }
 
     private fun asUser(registerRequest: RegisterRequest) = DbUser(
@@ -169,6 +182,7 @@ open class SecurityService(
         orcid = registerRequest.orcid,
         secret = securityUtil.newKey(),
         notificationsEnabled = registerRequest.notificationsEnabled,
+        storageMode = securityProps.filesProperties.defaultMode,
         passwordDigest = securityUtil.getPasswordDigest(registerRequest.password)
     )
 }

submission/submission-security/src/test/kotlin/ebi/ac/uk/security/service/ProfileServiceTest.kt

@@ -1,9 +1,10 @@
 package ebi.ac.uk.security.service
 
+import ac.uk.ebi.biostd.common.properties.StorageMode
 import ac.uk.ebi.biostd.persistence.model.DbUser
 import ac.uk.ebi.biostd.persistence.model.DbUserGroup
-import ebi.ac.uk.security.integration.model.api.GroupMagicFolder
-import ebi.ac.uk.security.integration.model.api.MagicFolder
+import ebi.ac.uk.security.integration.model.api.GroupFolder
+import ebi.ac.uk.security.integration.model.api.NfsUserFolder
 import ebi.ac.uk.security.integration.model.api.SecurityUser
 import io.github.glytching.junit.extension.folder.TemporaryFolder
 import io.github.glytching.junit.extension.folder.TemporaryFolderExtension
@@ -26,19 +27,20 @@ class ProfileServiceTest(temporaryFolder: TemporaryFolder) {
         groups = mutableSetOf(testGroup),
         passwordDigest = "".toByteArray(),
         superuser = true,
+        storageMode = StorageMode.NFS,
         notificationsEnabled = true
     )
 
     private val testInstance = ProfileService(filesDir)
 
     @Test
     fun getUserProfile() {
-        val expectedUserFolder = MagicFolder(
+        val expectedUserFolder = NfsUserFolder(
             relativePath = Paths.get("69/214a2f-f80b-4f33-86b7-26d3bd0453aa-a3"),
             path = Paths.get("$filesDir/69/214a2f-f80b-4f33-86b7-26d3bd0453aa-a3")
         )
 
-        val expectedGroupFolder = GroupMagicFolder(
+        val expectedGroupFolder = GroupFolder(
             groupName = "Test Group",
             description = "Test Group Description",
             path = Paths.get("$filesDir/fd/9f87b3-9de8-4036-be7a-3ac8cbc44ddd-b0")
@@ -52,7 +54,7 @@ class ProfileServiceTest(temporaryFolder: TemporaryFolder) {
             orcid = "0000-0002-1825-0097",
             secret = "69214a2f-f80b-4f33-86b7-26d3bd0453aa",
             superuser = true,
-            magicFolder = expectedUserFolder,
+            userFolder = expectedUserFolder,
             groupsFolders = listOf(expectedGroupFolder),
             permissions = emptySet(),
             notificationsEnabled = true

submission/submission-security/src/test/kotlin/ebi/ac/uk/security/service/SecurityQueryServiceTest.kt

@@ -1,5 +1,8 @@
 package ebi.ac.uk.security.service
 
+import ac.uk.ebi.biostd.common.properties.FilesProperties
+import ac.uk.ebi.biostd.common.properties.SecurityProperties
+import ac.uk.ebi.biostd.common.properties.StorageMode
 import ac.uk.ebi.biostd.persistence.model.DbUser
 import ac.uk.ebi.biostd.persistence.repositories.UserDataRepository
 import ebi.ac.uk.security.integration.exception.UserNotFoundByEmailException
@@ -23,9 +26,10 @@ import org.junit.jupiter.api.extension.ExtendWith
 class SecurityQueryServiceTest(
     @MockK private val securityUtil: SecurityUtil,
     @MockK private val profileService: ProfileService,
-    @MockK private val userRepository: UserDataRepository
+    @MockK private val userRepository: UserDataRepository,
+    @MockK private val securityProperties: SecurityProperties,
 ) {
-    private val testInstance = SecurityQueryService(securityUtil, profileService, userRepository)
+    private val testInstance = SecurityQueryService(securityUtil, profileService, userRepository, securityProperties)
 
     @AfterEach
     fun afterEach() = clearAllMocks()
@@ -49,7 +53,7 @@ class SecurityQueryServiceTest(
     @Test
     fun `get user`(
         @MockK dbUser: DbUser,
-        @MockK securityUser: SecurityUser
+        @MockK securityUser: SecurityUser,
     ) {
         every { profileService.asSecurityUser(dbUser) } returns securityUser
         every { userRepository.findByEmail("user@test.org") } returns dbUser
@@ -68,7 +72,7 @@ class SecurityQueryServiceTest(
     @Test
     fun `get user profile`(
         @MockK dbUser: DbUser,
-        @MockK userInfo: UserInfo
+        @MockK userInfo: UserInfo,
     ) {
         every { profileService.getUserProfile(dbUser, "the-token") } returns userInfo
         every { securityUtil.checkToken("the-token") } returns dbUser
@@ -87,7 +91,7 @@ class SecurityQueryServiceTest(
     @Test
     fun `get or create when the user exists`(
         @MockK dbUser: DbUser,
-        @MockK securityUser: SecurityUser
+        @MockK securityUser: SecurityUser,
     ) {
         every { profileService.asSecurityUser(dbUser) } returns securityUser
         every { userRepository.findByEmail("user@test.org") } returns dbUser
@@ -98,9 +102,12 @@ class SecurityQueryServiceTest(
     @Test
     fun `get or create when the user does not exist`(
         @MockK dbUser: DbUser,
-        @MockK securityUser: SecurityUser
+        @MockK securityUser: SecurityUser,
+        @MockK fileProperties: FilesProperties,
     ) {
         val dbUserSlot = slot<DbUser>()
+        every { securityProperties.filesProperties } returns fileProperties
+        every { fileProperties.defaultMode } returns StorageMode.NFS
         every { securityUtil.newKey() } returns "a-new-key"
         every { userRepository.save(capture(dbUserSlot)) } returns dbUser
         every { profileService.asSecurityUser(dbUser) } returns securityUser
@@ -113,7 +120,7 @@ class SecurityQueryServiceTest(
     @Test
     fun `get or register when the user exists`(
         @MockK dbUser: DbUser,
-        @MockK securityUser: SecurityUser
+        @MockK securityUser: SecurityUser,
     ) {
         every { profileService.asSecurityUser(dbUser) } returns securityUser
         every { userRepository.findByEmail("user@test.org") } returns dbUser