Pivotal ID # 182800115: Folder Submissions handling

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/api/FireWebClient.kt

@@ -6,6 +6,7 @@ import org.springframework.http.HttpHeaders
 import org.springframework.http.HttpStatus.NOT_FOUND
 import org.springframework.util.LinkedMultiValueMap
 import org.springframework.web.client.HttpClientErrorException
+import org.springframework.web.client.RestOperations
 import org.springframework.web.client.RestTemplate
 import org.springframework.web.client.getForObject
 import org.springframework.web.client.postForObject
@@ -65,15 +66,10 @@ internal class FireWebClient(
     override fun findByMd5(md5: String): List<FireApiFile> =
         template.getForObject<Array<FireApiFile>>("/objects/md5/$md5").toList()
 
-    override fun findByPath(path: String): FireApiFile? {
-        return runCatching { template.getForObject<FireApiFile>("/objects/path/$path") }
-            .getOrElse { if (it is ClientException && it.statusCode == NOT_FOUND) return null else throw it }
-    }
+    override fun findByPath(path: String): FireApiFile? = template.getForObjectOrNull("/objects/path/$path")
 
-    override fun findAllInPath(path: String): List<FireApiFile> {
-        return runCatching { template.getForObject<Array<FireApiFile>>("/objects/entries/path/$path").toList() }
-            .getOrElse { if (it is ClientException && it.statusCode == NOT_FOUND) return emptyList() else throw it }
-    }
+    override fun findAllInPath(path: String): List<FireApiFile> =
+        template.getForObjectOrNull<Array<FireApiFile>>("/objects/entries/path/$path").orEmpty().toList()
 
     override fun publish(fireOid: String) {
         template.put("/objects/$fireOid/publish", null)
@@ -83,3 +79,11 @@ internal class FireWebClient(
         template.delete("/objects/$fireOid/publish")
     }
 }
+
+/**
+ * Perform same as @see [RestOperations.getForObject] but maps 404 status response into null result.
+ */
+private inline fun <reified T> RestOperations.getForObjectOrNull(url: String, vararg uriVariables: Any): T? {
+    val result = runCatching { getForObject<T>(url, *uriVariables) }
+    return result.getOrElse { if (it is ClientException && it.statusCode == NOT_FOUND) null else throw it }
+}

client/fire-webclient/src/main/kotlin/uk/ac/ebi/fire/client/model/FireApiFile.kt

@@ -4,7 +4,7 @@ data class FireApiFile(
     val objectId: Number,
     val fireOid: String,
     val objectMd5: String,
-    val objectSize: Number,
+    val objectSize: Long,
     val createTime: String,
     val filesystemEntry: FileSystemEntry? = null,
 )
@@ -21,10 +21,3 @@ data class FileSystemEntry(
     val path: String?,
     val published: Boolean,
 )
-
-data class MetadataEntry(
-    val key: String,
-    val value: String,
-) {
-    override fun toString(): String = "\"${key}\": \"${value}\""
-}

commons/commons-model-extended/src/main/kotlin/ebi/ac/uk/extended/model/ExtendedModel.kt

@@ -64,10 +64,8 @@ data class NfsFile(
     override val md5: String,
     override val size: Long,
     override val attributes: List<ExtAttribute> = listOf(),
-) : ExtFile() {
-    override val type: ExtFileType
-        get() = if (file.isDirectory) DIR else FILE
-}
+    override val type: ExtFileType = if (file.isDirectory) DIR else FILE,
+) : ExtFile()
 
 data class ExtFileList(
     val filePath: String,

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt

@@ -36,7 +36,7 @@ object FileUtils {
     fun copyOrReplaceFile(
         source: File,
         target: File,
-        permissions: Permissions
+        permissions: Permissions,
     ) = when (isDirectory(source)) {
         true -> FileUtilsHelper.copyFolder(source.toPath(), target.toPath(), permissions)
         false -> FileUtilsHelper.copyFile(source.toPath(), target.toPath(), permissions)
@@ -45,12 +45,12 @@ object FileUtils {
     fun copyOrReplaceFile(
         source: InputStream,
         target: File,
-        permissions: Permissions
+        permissions: Permissions,
     ) = FileUtilsHelper.copyFile(source, target.toPath(), permissions)
 
     fun getOrCreateFolder(
         folder: Path,
-        permissions: Set<PosixFilePermission>
+        permissions: Set<PosixFilePermission>,
     ): Path {
         require(exists(folder).not() || isDirectory(folder.toFile())) { "'$folder' points to a file" }
         createFolderIfNotExist(folder, permissions)
@@ -82,7 +82,7 @@ object FileUtils {
     fun moveFile(
         source: File,
         target: File,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         deleteFile(target)
         when (isDirectory(source)) {
@@ -94,7 +94,7 @@ object FileUtils {
     fun createHardLink(
         source: File,
         target: File,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         when (isDirectory(source)) {
             true -> createFolderHardLinks(source.toPath(), target.toPath(), permissions)
@@ -122,6 +122,17 @@ object FileUtils {
 
     fun md5(file: File): String = if (file.isFile) calculateMd5(file) else ""
 
+    fun md5(value: String): String = DigestUtils.md5Hex(value).uppercase()
+
+    fun listAllFiles(file: File): List<File> {
+        require(file.isDirectory) { "$file need to be a directory" }
+        return Files.walk(file.toPath())
+            .skip(1)
+            .sorted()
+            .map { it.toFile() }
+            .toList()
+    }
+
     fun listFiles(file: File): List<File> =
         if (isDirectory(file)) Files.list(file.toPath()).map { it.toFile() }.toList() else emptyList()
 
@@ -137,7 +148,7 @@ internal object FileUtilsHelper {
     fun createFolderHardLinks(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         deleteFolder(target)
         Files.walkFileTree(source, HardLinkFileVisitor(source, target, permissions))
@@ -146,7 +157,7 @@ internal object FileUtilsHelper {
     fun createFileHardLink(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         deleteFolder(target)
         Files.createLink(source, createParentDirectories(target, permissions.folder))
@@ -161,7 +172,7 @@ internal object FileUtilsHelper {
     fun copyFolder(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         deleteFolder(target)
         Files.walkFileTree(source, CopyFileVisitor(source, target, permissions))
@@ -170,7 +181,7 @@ internal object FileUtilsHelper {
     fun moveFolder(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         deleteFolder(target)
         Files.walkFileTree(source, MoveFileVisitor(source, target, permissions))
@@ -180,7 +191,7 @@ internal object FileUtilsHelper {
     fun copyFile(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         Files.copy(source, createParentDirectories(target, permissions.folder), REPLACE_EXISTING)
         Files.setPosixFilePermissions(target, permissions.file)
@@ -189,7 +200,7 @@ internal object FileUtilsHelper {
     fun copyFile(
         source: InputStream,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         Files.copy(source, createParentDirectories(target, permissions.folder), REPLACE_EXISTING)
         Files.setPosixFilePermissions(target, permissions.file)
@@ -198,7 +209,7 @@ internal object FileUtilsHelper {
     fun moveFile(
         source: Path,
         target: Path,
-        permissions: Permissions
+        permissions: Permissions,
     ) {
         Files.move(source, createParentDirectories(target, permissions.folder), REPLACE_EXISTING)
         Files.setPosixFilePermissions(target, permissions.file)

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt

@@ -14,6 +14,8 @@ fun File.notExist() = Files.exists(toPath()).not()
 
 fun File.asFileList(): List<File> = FileUtils.listFiles(this)
 
+fun File.allSubFiles(): List<File> = FileUtils.listAllFiles(this)
+
 fun File.size() = FileUtils.size(this)
 
 fun File.md5() = FileUtils.md5(this)

commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt

@@ -2,6 +2,7 @@ package ebi.ac.uk.io
 
 import ebi.ac.uk.io.FileUtilsHelper.createFolderIfNotExist
 import ebi.ac.uk.io.ext.createDirectory
+import ebi.ac.uk.io.ext.createFile
 import ebi.ac.uk.io.ext.createNewFile
 import ebi.ac.uk.io.ext.newFile
 import ebi.ac.uk.test.clean
@@ -259,6 +260,20 @@ internal class FileUtilsTest(private val temporaryFolder: TemporaryFolder) {
             assertThat(FileUtils.md5(file)).isEqualTo("FC5D029EE5D34A268F8FA016E949073B")
         }
 
+        @Test
+        fun listAllFiles() {
+            val folder = temporaryFolder.createDirectory("list-all-test")
+            val dir1 = folder.createDirectory("dir1")
+            val dir2 = folder.createDirectory("dir2")
+            val file3 = folder.createFile("file3", "file3")
+            val dir11 = dir1.createDirectory("dir11")
+            val file11a = dir11.createFile("file11a", "11a-content")
+            val file11b = dir11.createFile("file11b", "11b-content")
+
+            val files = FileUtils.listAllFiles(folder)
+            assertThat(files).containsExactly(dir1, dir11, file11a, file11b, dir2, file3)
+        }
+
         @Nested
         inner class Md5Performance {
             lateinit var file: File

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireService.kt

@@ -6,7 +6,8 @@ import ebi.ac.uk.extended.model.ExtFileType.FILE
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.extended.model.FireFile
 import ebi.ac.uk.extended.model.NfsFile
-import org.zeroturnaround.zip.ZipUtil
+import ebi.ac.uk.io.ext.md5
+import ebi.ac.uk.io.ext.size
 import uk.ac.ebi.fire.client.integration.web.FireClient
 import java.io.File
 import java.nio.file.Files
@@ -29,28 +30,49 @@ class FireService(
      * different path) and if so, even if the file exists in FIRE, it gets duplicated to ensure consistency.
      */
     fun getOrPersist(sub: ExtSubmission, file: ExtFile): FirePersistResult {
-        val expectedPath = "/${sub.relPath}/${file.relPath}"
         return when (file) {
-            is FireFile -> fromFireFile(file, expectedPath)
-            is NfsFile -> when (file.type) {
-                FILE -> fromNfsFile(file, expectedPath)
-                DIR -> fromNfsFile(file.copy(file = compress(sub, file.file)), "$expectedPath.zip")
+            is FireFile -> {
+                val downloadFile = { client.downloadByFireId(file.fireId, file.fileName) }
+                reuseOrPersistFireFile(file, sub.relPath, downloadFile)
+            }
+            is NfsFile -> {
+                val nfsFile = if (file.type == FILE) file else asCompressedFile(sub, file)
+                val downloadFile = { nfsFile.file }
+                return reuseOrPersistFireFile(nfsFile, sub.relPath, downloadFile)
             }
         }
     }
 
-    private fun fromNfsFile(file: NfsFile, expectedPath: String): FirePersistResult =
-        reuseOrPersistFireFile(file, expectedPath) { file.file }
+    private fun asCompressedFile(sub: ExtSubmission, directory: NfsFile): NfsFile {
+        fun compress(file: File): File {
+            val tempFolder = fireTempDirPath.resolve("${sub.accNo}/${sub.version}")
+            tempFolder.mkdirs()
+
+            val target = tempFolder.resolve("${file.name}.zip")
+            Files.deleteIfExists(target.toPath())
+            ZipUtil.pack(file, target)
+            return target
+        }
 
-    private fun fromFireFile(file: FireFile, expectedPath: String): FirePersistResult =
-        reuseOrPersistFireFile(file, expectedPath) { client.downloadByFireId(file.fireId, file.fileName) }
+        val compressed = compress(directory.file)
+        return directory.copy(
+            filePath = "${directory.filePath}.zip",
+            relPath = "${directory.relPath}.zip",
+            file = compressed,
+            fullPath = "${directory.fullPath}.zip",
+            md5 = compressed.md5(),
+            size = compressed.size(),
+            type = DIR
+        )
+    }
 
     @Suppress("ReturnCount")
     private fun reuseOrPersistFireFile(
         file: ExtFile,
-        expectedPath: String,
+        subRelpath: String,
         fallbackFile: () -> File,
     ): FirePersistResult {
+        val expectedPath = "/$subRelpath/${file.relPath}"
         val files = client.findByMd5(file.md5)
 
         val noPath = files.firstOrNull { it.filesystemEntry?.path == null }
@@ -63,22 +85,11 @@ class FireService(
         return FirePersistResult(setMetadata(saved.fireOid, file, expectedPath), true)
     }
 
-    private fun setMetadata(fireOid: String, file: ExtFile, path: String): FireFile {
-        client.setPath(fireOid, path)
+    private fun setMetadata(fireOid: String, file: ExtFile, expectedPath: String): FireFile {
+        client.setPath(fireOid, expectedPath)
         return asFireFile(file, fireOid)
     }
 
-    private fun compress(sub: ExtSubmission, file: File): File {
-        val tempFolder = fireTempDirPath.resolve("${sub.accNo}/${sub.version}")
-        tempFolder.mkdirs()
-
-        val target = tempFolder.resolve(file.name)
-        Files.deleteIfExists(target.toPath())
-        ZipUtil.pack(file, target)
-
-        return target
-    }
-
     private fun asFireFile(file: ExtFile, fireId: String): FireFile = FireFile(
         filePath = file.filePath,
         relPath = file.relPath,

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/ZipUtil.kt

@@ -0,0 +1,34 @@
+package ac.uk.ebi.biostd.persistence.filesystem.fire
+
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import kotlin.io.path.isDirectory
+
+object ZipUtil {
+
+    fun pack(sourceDir: File, zipFile: File) {
+        ZipOutputStream(zipFile.outputStream()).use { zs ->
+            val sourcePath = sourceDir.toPath()
+            Files.walk(sourcePath)
+                .filter { path -> path.isDirectory().not() }
+                .forEach { path ->
+                    zs.putNextEntry(createZipEntry(path, sourcePath))
+                    Files.copy(path, zs)
+                    zs.closeEntry()
+                }
+        }
+    }
+
+    fun unpack(zip: File, outputDir: File) {
+        org.zeroturnaround.zip.ZipUtil.unpack(zip, outputDir)
+    }
+
+    private fun createZipEntry(filePath: Path, sourcePath: Path): ZipEntry {
+        val zipEntry = ZipEntry(sourcePath.relativize(filePath).toString())
+        zipEntry.time = 0
+        return zipEntry
+    }
+}