Pivotal ID # 177839764: FIRE FileProcessingService

buildSrc/src/main/kotlin/Dependencies.kt

@@ -214,6 +214,7 @@ object Projects {
     const val CommonsModelExtendedMapping = ":commons:commons-model-extended-mapping"
     const val CommonsModelExtendedSerialization = ":commons:commons-model-extended-serialization"
     const val CommonsTest = ":commons:commons-test"
+    const val FireWebClient = ":client:fire-webclient"
     const val SubmissionConfig = ":submission:submission-config"
     const val SubmissionPersistenceCommonApi = ":submission:persistence-common-api"
     const val SubmissionPersistenceFilesystem = ":submission:persistence-filesystem"

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/api/FireClient.kt

@@ -42,6 +42,10 @@ internal class FireClient(
         return template.put("$FIRE_OBJECTS_URL/$fireOid/firePath", HttpEntity(null, headers))
     }
 
+    override fun unsetPath(fireOid: String) {
+        template.delete("$FIRE_OBJECTS_URL/$fireOid/firePath")
+    }
+
     override fun downloadByPath(path: String): File {
         val tmpFile = File(tmpDirPath, path.substringAfterLast("/"))
         val fileContent = template.getForObject<ByteArray>("$FIRE_OBJECTS_URL/blob/path/$path")
@@ -50,6 +54,9 @@ internal class FireClient(
         return tmpFile
     }
 
+    override fun findAllInPath(path: String): List<FireFile> =
+        template.getForObject("$FIRE_OBJECTS_URL/entries/path/$path")
+
     override fun publish(fireOid: String) {
         template.put("$FIRE_OBJECTS_URL/$fireOid/publish", null)
     }

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/integration/web/FireOperations.kt

@@ -8,8 +8,12 @@ interface FireOperations {
 
     fun setPath(fireOid: String, path: String)
 
+    fun unsetPath(fireOid: String)
+
     fun downloadByPath(path: String): File
 
+    fun findAllInPath(path: String): List<FireFile>
+
     fun publish(fireOid: String)
 
     fun unpublish(fireOid: String)

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/model/FireFile.kt

@@ -6,5 +6,5 @@ data class FireFile(
     val objectMd5: String,
     val objectSize: Number,
     val createTime: String,
-    val filesystemEntry: FileSystemEntry?
+    val filesystemEntry: FileSystemEntry? = null
 )

client/fire-webclient/src/test/kotlin/uk/ac/ebi/fire/client/api/FireClientTest.kt

@@ -61,6 +61,15 @@ class FireClientTest(
         verify(exactly = 1) { template.put("$FIRE_OBJECTS_URL/the-fire-oid/firePath", httpEntity) }
     }
 
+    @Test
+    fun `unset path`() {
+        every { template.delete("$FIRE_OBJECTS_URL/the-fire-oid/firePath") } answers { nothing }
+
+        testInstance.unsetPath("the-fire-oid")
+
+        verify(exactly = 1) { template.delete("$FIRE_OBJECTS_URL/the-fire-oid/firePath") }
+    }
+
     @Test
     fun `download by path`() {
         val file = tmpFolder.createFile("test.txt", "test content")
@@ -78,6 +87,20 @@ class FireClientTest(
         }
     }
 
+    @Test
+    fun `find all by path`(@MockK fireFile: FireFile) {
+        every {
+            template.getForObject("$FIRE_OBJECTS_URL/entries/path/my/path", List::class.java)
+        } returns listOf(fireFile)
+
+        val files = testInstance.findAllInPath("my/path")
+        assertThat(files).hasSize(1)
+        assertThat(files.first()).isEqualTo(fireFile)
+        verify(exactly = 1) {
+            template.getForObject("$FIRE_OBJECTS_URL/entries/path/my/path", List::class.java)
+        }
+    }
+
     @Test
     fun publish() {
         every { template.put("$FIRE_OBJECTS_URL/the-fire-oid/publish", null) } answers { nothing }

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt

@@ -28,7 +28,3 @@ fun File.createNewFile(name: String, text: String): File {
 fun File.gZipTo(target: File) {
     FileOutputStream(target).use { GZIPOutputStream(it).bufferedWriter().use { writer -> writer.write(readText()) } }
 }
-
-fun File.copyTo(target: File) {
-    FileOutputStream(target).use { writer -> writer.write(readBytes()) }
-}

submission/persistence-filesystem/build.gradle.kts

@@ -7,6 +7,7 @@ import Projects.CommonsModelExtended
 import Projects.CommonsModelExtendedMapping
 import Projects.CommonsSerialization
 import Projects.CommonsUtil
+import Projects.FireWebClient
 import Projects.SubmissionPersistenceCommonApi
 import TestDependencies.BaseTestCompileDependencies
 import TestDependencies.BaseTestRuntimeDependencies
@@ -17,6 +18,7 @@ dependencies {
     api(project(CommonsModelExtended))
     api(project(CommonsModelExtendedMapping))
     api(project(CommonsUtil))
+    api(project(FireWebClient))
     api(project(SubmissionPersistenceCommonApi))
 
     implementation(Arrow)

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt

@@ -1,8 +1,8 @@
 package ac.uk.ebi.biostd.persistence.filesystem.api
 
+import ac.uk.ebi.biostd.persistence.filesystem.request.FilePersistenceRequest
 import ebi.ac.uk.extended.model.ExtSubmission
-import ebi.ac.uk.extended.model.FileMode
 
 interface FilesService {
-    fun persistSubmissionFiles(submission: ExtSubmission, mode: FileMode): ExtSubmission
+    fun persistSubmissionFiles(request: FilePersistenceRequest): ExtSubmission
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FtpService.kt

@@ -5,5 +5,5 @@ import ebi.ac.uk.extended.model.ExtSubmission
 interface FtpService {
     fun processSubmissionFiles(submission: ExtSubmission)
 
-    fun createFtpFolder(relPath: String)
+    fun generateFtpLinks(accNo: String)
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/extensions/FilePermissionsExtensions.kt

@@ -0,0 +1,14 @@
+package ac.uk.ebi.biostd.persistence.filesystem.extensions
+
+import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.io.RWXR_XR_X
+import ebi.ac.uk.io.RWXR_X___
+import ebi.ac.uk.io.RW_R__R__
+import ebi.ac.uk.io.RW_R_____
+import java.nio.file.attribute.PosixFilePermission
+
+object FilePermissionsExtensions {
+    fun ExtSubmission.filePermissions(): Set<PosixFilePermission> = if (released) RW_R__R__ else RW_R_____
+
+    fun ExtSubmission.folderPermissions(): Set<PosixFilePermission> = if (released) RWXR_XR_X else RWXR_X___
+}

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt

@@ -1,11 +1,51 @@
 package ac.uk.ebi.biostd.persistence.filesystem.fire
 
 import ac.uk.ebi.biostd.persistence.filesystem.api.FilesService
+import ac.uk.ebi.biostd.persistence.filesystem.request.FilePersistenceRequest
+import ac.uk.ebi.biostd.persistence.filesystem.request.Md5
+import ac.uk.ebi.biostd.persistence.filesystem.service.processFiles
+import ebi.ac.uk.extended.model.ExtFile
 import ebi.ac.uk.extended.model.ExtSubmission
-import ebi.ac.uk.extended.model.FileMode
+import ebi.ac.uk.extended.model.FireFile
+import ebi.ac.uk.extended.model.NfsFile
+import mu.KotlinLogging
+import uk.ac.ebi.fire.client.integration.web.FireWebClient
 
-class FireFilesService : FilesService {
-    override fun persistSubmissionFiles(submission: ExtSubmission, mode: FileMode): ExtSubmission {
-        TODO("Not yet implemented")
+private val logger = KotlinLogging.logger {}
+
+class FireFilesService(
+    private val fireWebClient: FireWebClient
+) : FilesService {
+    override fun persistSubmissionFiles(request: FilePersistenceRequest): ExtSubmission {
+        val (submission, _, previousFiles) = request
+        logger.info { "Starting processing files of submission ${submission.accNo} over FIRE" }
+        val config = FireFileProcessingConfig(submission.relPath, fireWebClient, previousFiles)
+        val processed = processFiles(submission) { config.processFile(it) }
+        logger.info { "Finishing processing files of submission ${submission.accNo} over FIRE" }
+        return processed
     }
 }
+
+data class FireFileProcessingConfig(
+    val relPath: String,
+    val fireWebClient: FireWebClient,
+    val previousFiles: Map<Md5, ExtFile>
+)
+
+fun FireFileProcessingConfig.processFile(file: ExtFile): ExtFile {
+    return if (file is NfsFile) processNfsFile(file) else file
+}
+
+fun FireFileProcessingConfig.processNfsFile(nfsFile: NfsFile): FireFile {
+    logger.info { "processing file ${nfsFile.fileName}" }
+    val fileFire = previousFiles[nfsFile.md5] as FireFile?
+    return if (fileFire == null) saveFile(nfsFile) else reusePreviousFile(fileFire, nfsFile)
+}
+
+private fun reusePreviousFile(fireFile: FireFile, nfsFile: NfsFile) =
+    FireFile(nfsFile.fileName, fireFile.fireId, fireFile.md5, fireFile.size, nfsFile.attributes)
+
+private fun FireFileProcessingConfig.saveFile(nfsFile: NfsFile): FireFile {
+    val store = fireWebClient.save(nfsFile.file, nfsFile.md5)
+    return FireFile(nfsFile.fileName, store.fireOid, store.objectMd5, store.objectSize.toLong(), nfsFile.attributes)
+}

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFtpService.kt

@@ -1,14 +1,46 @@
 package ac.uk.ebi.biostd.persistence.filesystem.fire
 
+import ac.uk.ebi.biostd.persistence.common.service.SubmissionQueryService
 import ac.uk.ebi.biostd.persistence.filesystem.api.FtpService
 import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.extended.model.FireFile
+import ebi.ac.uk.extended.model.allFiles
+import uk.ac.ebi.fire.client.integration.web.FireWebClient
 
-class FireFtpService : FtpService {
+class FireFtpService(
+    private val fireWebClient: FireWebClient,
+    private val submissionQueryService: SubmissionQueryService
+) : FtpService {
     override fun processSubmissionFiles(submission: ExtSubmission) {
-        TODO("Not yet implemented")
+        cleanFtpFolder(submission.relPath)
+        if (submission.released) publishFiles(submission)
     }
 
-    override fun createFtpFolder(relPath: String) {
-        TODO("Not yet implemented")
+    override fun generateFtpLinks(accNo: String) {
+        val submission = submissionQueryService.getExtByAccNo(accNo)
+        cleanFtpFolder(submission.relPath)
+        publishFiles(submission)
+    }
+
+    private fun publishFiles(submission: ExtSubmission) =
+        submission
+            .allFiles
+            .map { it as FireFile }
+            .forEach { publishFile(it, submission.relPath) }
+
+    private fun publishFile(file: FireFile, relPath: String) {
+        fireWebClient.setPath(file.fireId, "$relPath/${file.fileName}")
+        fireWebClient.publish(file.fireId)
+    }
+
+    private fun cleanFtpFolder(relPath: String) {
+        fireWebClient
+            .findAllInPath(relPath)
+            .forEach { unpublishFile(it.fireOid) }
+    }
+
+    private fun unpublishFile(fireId: String) {
+        fireWebClient.unpublish(fireId)
+        fireWebClient.unsetPath(fireId)
     }
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFileProcessingConfigExt.kt → submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFileProcessingConfig.kt

@@ -1,15 +1,25 @@
 package ac.uk.ebi.biostd.persistence.filesystem.nfs
 
-import ac.uk.ebi.biostd.persistence.filesystem.request.FileProcessingConfig
+import ebi.ac.uk.extended.model.FileMode
 import ebi.ac.uk.extended.model.NfsFile
 import ebi.ac.uk.io.FileUtils
 import ebi.ac.uk.io.ext.md5
 import ebi.ac.uk.io.ext.notExist
 import mu.KotlinLogging
+import java.io.File
+import java.nio.file.attribute.PosixFilePermission
 
 private val logger = KotlinLogging.logger {}
 
-fun FileProcessingConfig.nfsCopy(extFile: NfsFile): NfsFile {
+data class NfsFileProcessingConfig(
+    val mode: FileMode,
+    val subFolder: File,
+    val tempFolder: File,
+    val filePermissions: Set<PosixFilePermission>,
+    val dirPermissions: Set<PosixFilePermission>
+)
+
+fun NfsFileProcessingConfig.nfsCopy(extFile: NfsFile): NfsFile {
     val source = if (extFile.file.startsWith(subFolder)) tempFolder.resolve(extFile.fileName) else extFile.file
     val target = subFolder.resolve(extFile.fileName)
     val current = tempFolder.resolve(extFile.fileName)
@@ -29,7 +39,7 @@ fun FileProcessingConfig.nfsCopy(extFile: NfsFile): NfsFile {
     return extFile.copy(file = target)
 }
 
-fun FileProcessingConfig.nfsMove(extFile: NfsFile): NfsFile {
+fun NfsFileProcessingConfig.nfsMove(extFile: NfsFile): NfsFile {
     val source = if (extFile.file.startsWith(subFolder)) tempFolder.resolve(extFile.fileName) else extFile.file
     val target = subFolder.resolve(extFile.fileName)
 

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt

@@ -1,10 +1,10 @@
 package ac.uk.ebi.biostd.persistence.filesystem.nfs
 
 import ac.uk.ebi.biostd.persistence.filesystem.api.FilesService
-import ac.uk.ebi.biostd.persistence.filesystem.pagetab.PageTabService
-import ac.uk.ebi.biostd.persistence.filesystem.request.FileProcessingConfig
-import ac.uk.ebi.biostd.persistence.filesystem.request.PageTabRequest
-import ac.uk.ebi.biostd.persistence.filesystem.service.FileProcessingService.processFiles
+import ac.uk.ebi.biostd.persistence.filesystem.extensions.FilePermissionsExtensions.filePermissions
+import ac.uk.ebi.biostd.persistence.filesystem.extensions.FilePermissionsExtensions.folderPermissions
+import ac.uk.ebi.biostd.persistence.filesystem.request.FilePersistenceRequest
+import ac.uk.ebi.biostd.persistence.filesystem.service.processFiles
 import ebi.ac.uk.extended.model.ExtFile
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.extended.model.FileMode
@@ -16,10 +16,7 @@ import ebi.ac.uk.io.FileUtils.getOrCreateFolder
 import ebi.ac.uk.io.FileUtils.moveFile
 import ebi.ac.uk.io.FileUtils.reCreateFolder
 import ebi.ac.uk.io.RWXR_XR_X
-import ebi.ac.uk.io.RWXR_X___
 import ebi.ac.uk.io.RWX______
-import ebi.ac.uk.io.RW_R__R__
-import ebi.ac.uk.io.RW_R_____
 import ebi.ac.uk.paths.FILES_PATH
 import ebi.ac.uk.paths.SubmissionFolderResolver
 import mu.KotlinLogging
@@ -29,19 +26,18 @@ import java.nio.file.attribute.PosixFilePermission
 private val logger = KotlinLogging.logger {}
 
 class NfsFilesService(
-    private val pageTabService: PageTabService,
     private val folderResolver: SubmissionFolderResolver
 ) : FilesService {
-    override fun persistSubmissionFiles(submission: ExtSubmission, mode: FileMode): ExtSubmission {
-        logger.info { "Starting processing files of submission ${submission.accNo}" }
+    override fun persistSubmissionFiles(request: FilePersistenceRequest): ExtSubmission {
+        val (submission, mode, _) = request
+        logger.info { "Starting processing files of submission ${submission.accNo} over NFS" }
 
-        val filePermissions = filePermissions(submission.released)
-        val folderPermissions = folderPermissions(submission.released)
+        val filePermissions = submission.filePermissions()
+        val folderPermissions = submission.folderPermissions()
         val submissionFolder = getOrCreateSubmissionFolder(submission, folderPermissions)
 
         val processed = processAttachedFiles(mode, submission, submissionFolder, filePermissions, folderPermissions)
-        pageTabService.generatePageTab(PageTabRequest(processed, submissionFolder, filePermissions, folderPermissions))
-        logger.info { "Finishing processing files of submission ${submission.accNo}" }
+        logger.info { "Finishing processing files of submission ${submission.accNo} over NFS" }
 
         return processed
     }
@@ -63,24 +59,18 @@ class NfsFilesService(
             reCreateFolder(subFolder, folderPermissions)
         }
 
-        val config = FileProcessingConfig(subFilesPath, tempFolder, filePermissions, folderPermissions)
-
-        fun processNfsFile(file: ExtFile): ExtFile {
-            val nfsFile = file as NfsFile
-            return if (mode == COPY) config.nfsCopy(nfsFile) else config.nfsMove(nfsFile)
-        }
-
-        val processed = processFiles(submission, ::processNfsFile)
+        val config = NfsFileProcessingConfig(mode, subFilesPath, tempFolder, filePermissions, folderPermissions)
+        val processed = processFiles(submission) { config.processFile(it) }
 
         logger.info { "Finishing processing submission ${submission.accNo} files in $mode" }
         deleteFile(tempFolder)
-
         return processed
     }
 
-    private fun filePermissions(released: Boolean) = if (released) RW_R__R__ else RW_R_____
-
-    private fun folderPermissions(released: Boolean) = if (released) RWXR_XR_X else RWXR_X___
+    private fun NfsFileProcessingConfig.processFile(file: ExtFile): NfsFile {
+        val nfsFile = file as NfsFile
+        return if (mode == COPY) nfsCopy(nfsFile) else nfsMove(nfsFile)
+    }
 
     private fun getOrCreateSubmissionFolder(submission: ExtSubmission, permissions: Set<PosixFilePermission>): File {
         val submissionPath = folderResolver.getSubFolder(submission.relPath)