Pivotal ID # 186992651: NFS Private Submissions Persistence

commons/commons-bio/src/main/kotlin/ebi/ac/uk/paths/SubmissionFolderResolver.kt

@@ -1,18 +1,29 @@
 package ebi.ac.uk.paths
 
+import ebi.ac.uk.extended.model.ExtSubmission
 import java.nio.file.Path
 
 const val FILES_PATH = "Files"
 
+// TODO should this be an application property for security reasons?
+const val PRIVATE_PATH = ".private"
+
 class SubmissionFolderResolver(
     private val submissionFolder: Path,
     private val ftpFolder: Path
 ) {
-
     fun getSubmissionFtpFolder(submissionRelPath: String): Path = ftpFolder.resolve(submissionRelPath)
 
     fun getSubFolder(submissionRelPath: String): Path = submissionFolder.resolve(submissionRelPath)
 
+    fun getPrivateSubFolderRoot(secret: String): Path {
+        return submissionFolder.resolve("$PRIVATE_PATH/${secret.take(2)}")
+    }
+
+    fun getPrivateSubFolder(secret: String, relPath: String): Path {
+        return getPrivateSubFolderRoot(secret).resolve("${secret.substring(2)}/$relPath")
+    }
+
     fun getSubFilePath(relPath: String, fileName: String): Path =
         submissionFolder.resolve(relPath).resolve(FILES_PATH).resolve(escapeFileName(fileName))
 

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt

@@ -78,12 +78,10 @@ object FileUtils {
     }
 
     fun deleteEmptyDirectories(dir: File) {
-        dir
-            .walkBottomUp()
+        dir.walkBottomUp()
             .asSequence()
-            .takeWhile { it != dir }
-            .forEach { deleteEmptyDirectories(it) }
-        if (dir.isDirectory && dir.isEmpty()) Files.delete(dir.toPath())
+            .filter { it.isDirectory && it.isEmpty() }
+            .forEach { Files.delete(it.toPath()) }
     }
 
     fun moveFile(

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt

@@ -16,4 +16,6 @@ interface FileStorageService {
         sub: ExtSubmission,
         process: (Flow<ExtFile>) -> Flow<ExtFile> = { it },
     )
+
+    suspend fun deleteEmptyFolders(sub: ExtSubmission)
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt

@@ -60,8 +60,9 @@ class NfsFilesService(
     }
 
     private fun getOrCreateSubmissionFolder(submission: ExtSubmission, permissions: Set<PosixFilePermission>): File {
-        val submissionPath = folderResolver.getSubFolder(submission.relPath)
+        val submissionPath = folderResolver.getPrivateSubFolder(submission.secretKey, submission.relPath)
         FileUtils.createParentFolders(submissionPath, RWXR_XR_X)
+
         return getOrCreateFolder(submissionPath, permissions).toFile()
     }
 
@@ -80,8 +81,11 @@ class NfsFilesService(
         logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" }
     }
 
-    override suspend fun deleteEmptyFolders(current: ExtSubmission) = withContext(Dispatchers.IO) {
-        val subFolder = folderResolver.getSubFolder(current.relPath)
-        FileUtils.deleteEmptyDirectories(subFolder.toFile())
+    override suspend fun deleteEmptyFolders(submission: ExtSubmission) = withContext(Dispatchers.IO) {
+        val subFolder = folderResolver.getSubFolder(submission.relPath).toFile()
+        val privateSubFolder = folderResolver.getPrivateSubFolderRoot(submission.secretKey).toFile()
+
+        FileUtils.deleteEmptyDirectories(privateSubFolder)
+        FileUtils.deleteEmptyDirectories(subFolder.parentFile)
     }
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFtpService.kt

@@ -10,19 +10,18 @@ import ebi.ac.uk.io.RW_R__R__
 import ebi.ac.uk.paths.SubmissionFolderResolver
 import java.io.File
 
+// TODO these classes should be renamed to NfsReleaserService and FireReleaserService to decouple them from the FTP concept
 class NfsFtpService(
     private val folderResolver: SubmissionFolderResolver,
 ) : FtpService {
     override suspend fun releaseSubmissionFile(file: ExtFile, subRelPath: String): ExtFile {
         return synchronized(this) {
             val nfsFile = file as NfsFile
-            val ftpFolder = getFtpFolder(subRelPath).toPath()
             val subFolder = folderResolver.getSubFolder(subRelPath)
-            FileUtils.createHardLink(nfsFile.file, subFolder, ftpFolder, Permissions(RW_R__R__, RWXR_XR_X))
-            nfsFile
+            val releasedFile = subFolder.resolve(nfsFile.relPath).toFile()
+
+            FileUtils.moveFile(nfsFile.file, releasedFile, Permissions(RW_R__R__, RWXR_XR_X))
+            nfsFile.copy(fullPath = releasedFile.absolutePath, file = releasedFile)
         }
     }
-
-    private fun getFtpFolder(relPath: String): File =
-        FileUtils.getOrCreateFolder(folderResolver.getSubmissionFtpFolder(relPath), RWXR_XR_X).toFile()
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt

@@ -15,8 +15,6 @@ import mu.KotlinLogging
 import uk.ac.ebi.extended.serialization.service.ExtSerializationService
 import uk.ac.ebi.extended.serialization.service.filesFlow
 
-private val logger = KotlinLogging.logger {}
-
 @Suppress("LongParameterList")
 class StorageService(
     private val fireFtpService: FireFtpService,
@@ -58,7 +56,7 @@ class StorageService(
         deleteEmptyFolders(sub)
     }
 
-    private suspend fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) {
+    override suspend fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) {
         FIRE -> fireFilesService.deleteEmptyFolders(sub)
         NFS -> nfsFilesService.deleteEmptyFolders(sub)
     }

submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestFinalizer.kt

@@ -7,6 +7,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceS
 import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService
 import ebi.ac.uk.extended.model.ExtFile
 import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.extended.model.StorageMode.NFS
 import ebi.ac.uk.extended.model.storageMode
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.filter
@@ -40,17 +41,18 @@ class SubmissionRequestFinalizer(
         val sub = queryService.getExtByAccNo(accNo, includeFileListFiles = true)
         val previous = queryService.findLatestInactiveByAccNo(sub.accNo, includeFileListFiles = true)
         if (previous != null) deleteRemainingFiles(sub, previous)
+        if (sub.storageMode == NFS) storageService.deleteEmptyFolders(sub)
         return sub
     }
 
-    private suspend fun deleteRemainingFiles(current: ExtSubmission?, previous: ExtSubmission) {
+    private suspend fun deleteRemainingFiles(current: ExtSubmission, previous: ExtSubmission) {
         val subFiles = subFilesSet(current)
         val accNo = previous.accNo
         val owner = previous.owner
 
         fun deleteRemainingFiles(allFiles: Flow<ExtFile>): Flow<ExtFile> {
             return allFiles
-                .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode }
+                .filter { subFiles.contains(it.filePath).not() || it.storageMode != current.storageMode }
                 .withIndex()
                 .onEach { (i, file) -> logger.info { "$accNo $owner Deleting file $i, path='${file.filePath}'" } }
                 .map { it.value }

submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt

@@ -9,7 +9,6 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService
 import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestFilesPersistenceService
 import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService
 import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService
-import ac.uk.ebi.biostd.submission.exceptions.UnreleasedSubmissionException
 import ebi.ac.uk.extended.model.ExtFile
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.extended.model.FireFile
@@ -89,15 +88,6 @@ class SubmissionRequestReleaser(
         releaseSubmission(submission)
     }
 
-    /**
-     * Generates/refresh FTP links for a given submission.
-     */
-    suspend fun generateFtpLinks(accNo: String) {
-        val submission = queryService.getExtByAccNo(accNo, includeFileListFiles = true)
-        require(submission.released) { throw UnreleasedSubmissionException() }
-        releaseSubmissionFiles(submission)
-    }
-
     private suspend fun releaseFile(sub: ExtSubmission, idx: Int, file: ExtFile): ExtFile {
         logger.info { "${sub.accNo}, ${sub.owner} Started publishing file $idx - ${file.filePath}" }
         val releasedFile = fileStorageService.releaseSubmissionFile(file, sub.relPath, sub.storageMode)

submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt

@@ -8,7 +8,6 @@ import ebi.ac.uk.security.integration.model.api.SecurityUser
 import org.springframework.security.access.prepost.PreAuthorize
 import org.springframework.web.bind.annotation.DeleteMapping
 import org.springframework.web.bind.annotation.PathVariable
-import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.PutMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestMapping
@@ -22,11 +21,6 @@ class SubmissionOperationsResource(
     private val submissionsWebHandler: SubmissionsWebHandler,
     private val submissionReleaser: SubmissionRequestReleaser,
 ) {
-    @PostMapping("/ftp/generate")
-    suspend fun generateFtpLinks(@RequestParam("accNo", required = true) accNo: String) {
-        submissionReleaser.generateFtpLinks(accNo)
-    }
-
     @DeleteMapping("/{accNo}")
     suspend fun deleteSubmission(
         @BioUser user: SecurityUser,