Pivotal ID # 185392780: User FTP files API support (#755)

https://www.pivotaltracker.com/story/show/185392780
- Changed logic to transform FTP check file in NFS as process will be
executed in data mover

submission/file-sources/src/main/kotlin/uk/ac/ebi/io/builder/FilesSourceListBuilder.kt

@@ -56,8 +56,9 @@ class FilesSourceListBuilder(
         }
 
         if (folder is FtpUserFolder) {
-            val path = if (rootPath == null) folder.relativePath else folder.relativePath.resolve(rootPath)
-            sources.add(FtpSource(description, path, ftpClient))
+            val ftpUrl = if (rootPath == null) folder.relativePath else folder.relativePath.resolve(rootPath)
+            val nfsPath = if (rootPath == null) folder.path else folder.path.resolve(rootPath)
+            sources.add(FtpSource(description, ftpUrl = ftpUrl, nfsPath = nfsPath, ftpClient = ftpClient))
         }
     }
 

submission/file-sources/src/main/kotlin/uk/ac/ebi/io/sources/FtpSource.kt

@@ -7,13 +7,17 @@ import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.constants.FileFields
 import uk.ac.ebi.io.builder.createFile
 import java.io.File
-import java.nio.file.Files
 import java.nio.file.Path
-import kotlin.io.path.outputStream
 
+/**
+ *  Ftp source. Mix both ftp protocol to validate file presence and direct ftp mount point to access file content.
+ *  This seperation is necesary as files are check in backend instance with no access to FTP file system while
+ *  processing is executed in data mover which can access moint point.
+ */
 class FtpSource(
     override val description: String,
-    private val basePath: Path,
+    private val ftpUrl: Path,
+    private val nfsPath: Path,
     private val ftpClient: FtpClient,
 ) : FilesSource {
     override suspend fun getExtFile(path: String, type: String, attributes: List<Attribute>): ExtFile? {
@@ -22,15 +26,10 @@ class FtpSource(
     }
 
     private fun findFile(filePath: String): File? {
-        val ftpPath = basePath.resolve(filePath)
+        val ftpPath = ftpUrl.resolve(filePath)
         val files = ftpClient.listFiles(ftpPath)
         if (files.isEmpty()) return null
-
-        val fileName = filePath.substringAfterLast("/")
-        val target = Files.createTempFile("ftp-file", fileName)
-
-        target.outputStream().use { ftpClient.downloadFile(ftpPath, it) }
-        return target.toFile()
+        return nfsPath.resolve(filePath).toFile()
     }
 
     override suspend fun getFileList(path: String): File? {

submission/submission-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/SecurityProperties.kt

@@ -24,6 +24,7 @@ data class FilesProperties(
     val defaultMode: StorageMode,
     val filesDirPath: String,
     val magicDirPath: String,
+    val ftpDirPath: String,
     val ftpUser: String,
     val ftpPassword: String,
     val ftpUrl: String,

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/integration/SecurityModuleConfig.kt

@@ -57,8 +57,7 @@ class SecurityModuleConfig(
             props,
             profileService,
             captchaVerifier,
-            eventsPublisherService,
-            ftpClient
+            eventsPublisherService
         )
     }
 
@@ -94,8 +93,9 @@ class SecurityModuleConfig(
 
         fun profileService(props: SecurityProperties): ProfileService {
             return ProfileService(
-                Paths.get(props.filesProperties.filesDirPath),
-                props.environment
+                nfsUserFtpDirPath = Paths.get(props.filesProperties.ftpDirPath),
+                nfsUserFilesDirPath = Paths.get(props.filesProperties.filesDirPath),
+                environment = props.environment
             )
         }
     }

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/integration/model/api/SecurityUser.kt

@@ -24,10 +24,22 @@ data class SecurityPermission(val accessType: AccessType, val accessTag: String)
 
 sealed interface UserFolder {
     val relativePath: Path
+    val path: Path
 }
 
-data class FtpUserFolder(override val relativePath: Path) : UserFolder
-data class NfsUserFolder(override val relativePath: Path, val path: Path) : UserFolder
+data class FtpUserFolder(
+    /**
+     * The user ftp path. Used to access files through ftp protocol.
+     */
+    override val relativePath: Path,
+
+    /**
+     * The File system ftp path. Used to access ftp files as file system file.
+     */
+    override val path: Path,
+) : UserFolder
+
+data class NfsUserFolder(override val relativePath: Path, override val path: Path) : UserFolder
 
 fun NfsUserFolder.resolve(subPath: String): Path = path.resolve(subPath)
 

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/service/ProfileService.kt

@@ -15,7 +15,8 @@ import java.nio.file.Path
 import java.nio.file.Paths
 
 class ProfileService(
-    private val filesDirPath: Path,
+    private val nfsUserFtpDirPath: Path,
+    private val nfsUserFilesDirPath: Path,
     private val environment: String,
 ) {
     fun getUserProfile(user: DbUser, token: String): UserInfo = UserInfo(asSecurityUser(user), token)
@@ -40,17 +41,21 @@ class ProfileService(
     private fun groupsMagicFolder(groups: Set<DbUserGroup>): List<GroupFolder> =
         groups.map { GroupFolder(it.name, groupMagicFolder(it), it.description) }
 
-    private fun groupMagicFolder(it: DbUserGroup) = Paths.get("$filesDirPath/${magicPath(it.secret, it.id, "b")}")
+    private fun groupMagicFolder(it: DbUserGroup) =
+        Paths.get("$nfsUserFilesDirPath/${magicPath(it.secret, it.id, "b")}")
 
     private fun userMagicFolder(folderType: StorageMode, secret: String, id: Long): UserFolder {
         fun nfsFolder(): NfsUserFolder {
             val relativePath = magicPath(secret, id, "a")
-            return NfsUserFolder(Paths.get(relativePath), Paths.get("$filesDirPath/$relativePath"))
+            return NfsUserFolder(Paths.get(relativePath), Paths.get("$nfsUserFilesDirPath/$relativePath"))
         }
 
         fun ftpFolder(): FtpUserFolder {
             val relativePath = magicPath(secret, id, "a")
-            return FtpUserFolder(Paths.get("$environment/$relativePath"))
+            return FtpUserFolder(
+                relativePath = Paths.get("$environment/$relativePath"),
+                path = Paths.get("$nfsUserFtpDirPath/$environment/$relativePath"),
+            )
         }
 
         return when (folderType) {

submission/submission-security/src/main/kotlin/ebi/ac/uk/security/service/SecurityService.kt

@@ -14,7 +14,6 @@ import ebi.ac.uk.extended.events.SecurityNotification
 import ebi.ac.uk.extended.events.SecurityNotificationType.ACTIVATION
 import ebi.ac.uk.extended.events.SecurityNotificationType.ACTIVATION_BY_EMAIL
 import ebi.ac.uk.extended.events.SecurityNotificationType.PASSWORD_RESET
-import ebi.ac.uk.ftp.FtpClient
 import ebi.ac.uk.io.FileUtils
 import ebi.ac.uk.io.RWXRWX___
 import ebi.ac.uk.io.RWX__X___
@@ -49,7 +48,6 @@ open class SecurityService(
     private val profileService: ProfileService,
     private val captchaVerifier: CaptchaVerifier,
     private val eventsPublisherService: EventsPublisherService,
-    private val ftpClient: FtpClient,
 ) : ISecurityService {
     override fun login(request: LoginRequest): UserInfo {
         val user = userRepository.getActiveByLoginOrEmail(request.login)
@@ -168,14 +166,15 @@ open class SecurityService(
         }
     }
 
-    private fun createFtpMagicFolder(magicFolder: FtpUserFolder) {
-        ftpClient.createFolder(magicFolder.relativePath)
+    private fun createFtpMagicFolder(ftpFolder: FtpUserFolder) {
+        FileUtils.getOrCreateFolder(ftpFolder.path.parent, RWX__X___)
+        FileUtils.getOrCreateFolder(ftpFolder.path, RWXRWX___)
     }
 
-    private fun createNfsMagicFolder(email: String, magicFolder: NfsUserFolder) {
-        FileUtils.getOrCreateFolder(magicFolder.path.parent, RWX__X___)
-        FileUtils.getOrCreateFolder(magicFolder.path, RWXRWX___)
-        FileUtils.createSymbolicLink(symLinkPath(email), magicFolder.path, RWXRWX___)
+    private fun createNfsMagicFolder(email: String, nfsFolder: NfsUserFolder) {
+        FileUtils.getOrCreateFolder(nfsFolder.path.parent, RWX__X___)
+        FileUtils.getOrCreateFolder(nfsFolder.path, RWXRWX___)
+        FileUtils.createSymbolicLink(symLinkPath(email), nfsFolder.path, RWXRWX___)
     }
 
     private fun symLinkPath(userEmail: String): Path {

submission/submission-security/src/test/kotlin/ebi/ac/uk/security/service/ProfileServiceTest.kt

@@ -16,7 +16,8 @@ import java.nio.file.Paths
 @ExtendWith(TemporaryFolderExtension::class)
 class ProfileServiceTest(temporaryFolder: TemporaryFolder) {
     private val environment = "env-test"
-    private val filesDir = temporaryFolder.root.toPath()
+    private val filesDir = temporaryFolder.createDirectory("userFiles")
+    private val ftpFilesDir = temporaryFolder.createDirectory("ftpFiles")
     private val testGroup = DbUserGroup("Test Group", "Test Group Description", "fd9f87b3-9de8-4036-be7a-3ac8cbc44ddd")
 
     private val testUser = DbUser(
@@ -32,7 +33,11 @@ class ProfileServiceTest(temporaryFolder: TemporaryFolder) {
         notificationsEnabled = true
     )
 
-    private val testInstance = ProfileService(filesDir, environment)
+    private val testInstance = ProfileService(
+        nfsUserFtpDirPath = ftpFilesDir.toPath(),
+        nfsUserFilesDirPath = filesDir.toPath(),
+        environment
+    )
 
     @Test
     fun getUserProfile() {

submission/submission-security/src/test/kotlin/ebi/ac/uk/security/service/SecurityServiceTest.kt

@@ -77,10 +77,13 @@ internal class SecurityServiceTest(
         userRepository,
         securityUtil,
         securityProps,
-        ProfileService(temporaryFolder.root.toPath(), ENVIRONMENT),
+        ProfileService(
+            nfsUserFilesDirPath = temporaryFolder.createDirectory("nfsFile").toPath(),
+            nfsUserFtpDirPath = temporaryFolder.createDirectory("ftpFiles").toPath(),
+            environment = ENVIRONMENT
+        ),
         captchaVerifier,
-        eventsPublisherService,
-        ftpClient
+        eventsPublisherService
     )
 
     @Nested

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/itest/FtpServer.kt

@@ -14,6 +14,7 @@ import kotlin.io.path.absolutePathString
 
 class FtpServer(
     private val server: DefaultFtpServer,
+    val fileSystemDirectory: File,
 ) {
 
     fun start() {
@@ -61,8 +62,9 @@ class FtpServer(
 
             serverFactory.addListener(listenerName, listenerFactory.createListener())
             val server = serverFactory.createServer()
-            serverFactory.userManager.save(newUser(config.userName, config.password))
-            return FtpServer(server as DefaultFtpServer)
+            val user = newUser(config.userName, config.password)
+            serverFactory.userManager.save(user)
+            return FtpServer(server as DefaultFtpServer, File(user.homeDirectory))
         }
 
         private fun newUser(

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/itest/ITestListener.kt

@@ -65,6 +65,7 @@ class ITestListener : TestExecutionListener {
         System.setProperty("app.security.filesProperties.ftpPassword", ftpPassword)
         System.setProperty("app.security.filesProperties.ftpUrl", ftpServer.getUrl())
         System.setProperty("app.security.filesProperties.ftpPort", ftpServer.ftpPort.toString())
+        System.setProperty("app.security.filesProperties.ftpDirPath", ftpServer.fileSystemDirectory.absolutePath)
     }
 
     private fun fireSetup() {

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionApiTest.kt

@@ -34,7 +34,6 @@ import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatExceptionOfType
 import org.junit.jupiter.api.Assertions.assertThrows
 import org.junit.jupiter.api.BeforeAll
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty
@@ -335,7 +334,6 @@ class SubmissionApiTest(
     }
 
     @Test
-    @Disabled
     fun `16-12 User with Ftp based folder submission`() = runTest {
         securityTestService.ensureUserRegistration(FtpSuperUser)
         webClient = getWebClient(serverPort, FtpSuperUser)

submission/submission-webapp/src/main/resources/application.yml

@@ -100,6 +100,7 @@ app:
     fileProperties:
       filesDirPath: # Absolute path to the folder to be used for the user/groups files
       magicDirPath: #  Absolute path to create user magic folder for dropbox simple access
+      ftpDirPath: #  Absolute path of users ftp. Used to access file
       ftpUser: # the ftp user
       ftpPassword: # the ftp password
       ftpUrl: # the ftp url without port