Pivotal ID # 184822139: Delete Submission Folder After Resubmission (#…

…709)

* added clean empty folder operations

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt

@@ -6,6 +6,7 @@ import ebi.ac.uk.io.FileUtilsHelper.createFolderHardLinks
 import ebi.ac.uk.io.FileUtilsHelper.createFolderIfNotExist
 import ebi.ac.uk.io.FileUtilsHelper.createParentDirectories
 import ebi.ac.uk.io.FileUtilsHelper.createSymLink
+import ebi.ac.uk.io.ext.isEmpty
 import ebi.ac.uk.io.ext.notExist
 import ebi.ac.uk.io.ext.size
 import org.apache.commons.codec.digest.DigestUtils
@@ -72,6 +73,15 @@ object FileUtils {
         }
     }
 
+    fun deleteEmptyDirectories(dir: File) {
+        dir
+            .walkBottomUp()
+            .asSequence()
+            .takeWhile { it != dir }
+            .forEach { deleteEmptyDirectories(it) }
+        if (dir.isDirectory && dir.isEmpty()) Files.delete(dir.toPath())
+    }
+
     fun moveFile(
         source: File,
         target: File,

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt

@@ -14,6 +14,8 @@ fun File.notExist() = Files.exists(toPath()).not()
 
 fun File.listFilesOrEmpty(): List<File> = FileUtils.listFiles(this)
 
+fun File.isEmpty(): Boolean = Files.newDirectoryStream(this.toPath()).use { return it.iterator().hasNext().not() }
+
 fun File.allSubFiles(): List<File> = FileUtils.listAllFiles(this)
 
 fun File.size(calculateDirectories: Boolean = true) = FileUtils.size(this, calculateDirectories)

commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt

@@ -267,6 +267,40 @@ internal class FileUtilsTest(private val temporaryFolder: TemporaryFolder) {
         }
     }
 
+    @Nested
+    inner class DeleteEmptyFolders {
+        @Test
+        fun whenEmpty() {
+            val main = temporaryFolder.createDirectory("main")
+            val sub1 = main.createDirectory("sub1")
+            val sub2 = main.createDirectory("sub2")
+            sub1.createDirectory("sub1Sub1")
+            sub1.createDirectory("sub1Sub2")
+            sub2.createDirectory("sub2Sub1")
+
+            FileUtils.deleteEmptyDirectories(main)
+
+            assertThat(main).doesNotExist()
+        }
+
+        @Test
+        fun whenNotEmpty() {
+            val main = temporaryFolder.createDirectory("main")
+            val sub1 = main.createDirectory("sub1")
+            val sub2 = main.createDirectory("sub2")
+            val sub1Sub1 = sub1.createDirectory("sub1Sub1")
+            val file = sub1Sub1.createFile("one.txt", "text")
+
+            FileUtils.deleteEmptyDirectories(main)
+
+            assertThat(file).exists()
+            assertThat(sub1Sub1).exists()
+            assertThat(sub2).doesNotExist()
+            assertThat(sub1).exists()
+            assertThat(main).exists()
+        }
+    }
+
     @Nested
     inner class Utilities {
         @Test

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt

@@ -11,5 +11,8 @@ interface FileStorageService {
 
     fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile)
 
-    fun deleteSubmissionFiles(sub: ExtSubmission)
+    fun deleteSubmissionFiles(
+        sub: ExtSubmission,
+        process: (Sequence<ExtFile>) -> Sequence<ExtFile> = { it },
+    )
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt

@@ -9,4 +9,6 @@ internal interface FilesService {
     fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile)
 
     fun deleteFtpFile(sub: ExtSubmission, file: ExtFile)
+
+    fun deleteEmptyFolders(current: ExtSubmission)
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt

@@ -65,4 +65,8 @@ class FireFilesService(
     override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) {
         // No need to delete FTP links on FIRE as file deleting complete this
     }
+
+    override fun deleteEmptyFolders(current: ExtSubmission) {
+        // No need to delete FIRE empty bucket as they only exists as files are in them
+    }
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt

@@ -76,4 +76,9 @@ class NfsFilesService(
         FileUtils.deleteFile(subFolder.resolve(file.relPath).toFile())
         logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" }
     }
+
+    override fun deleteEmptyFolders(current: ExtSubmission) {
+        val subFolder = folderResolver.getSubFolder(current.relPath)
+        FileUtils.deleteEmptyDirectories(subFolder.toFile())
+    }
 }

submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt

@@ -49,7 +49,16 @@ class StorageService(
         }
     }
 
-    override fun deleteSubmissionFiles(sub: ExtSubmission) {
-        serializationService.fileSequence(sub).forEach { file -> deleteSubmissionFile(sub, file) }
+    override fun deleteSubmissionFiles(
+        sub: ExtSubmission,
+        process: (Sequence<ExtFile>) -> Sequence<ExtFile>,
+    ) {
+        process(serializationService.fileSequence(sub)).forEach { file -> deleteSubmissionFile(sub, file) }
+        deleteEmptyFolders(sub)
+    }
+
+    private fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) {
+        FIRE -> fireFilesService.deleteEmptyFolders(sub)
+        NFS -> nfsFilesService.deleteEmptyFolders(sub)
     }
 }

submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt

@@ -31,17 +31,19 @@ class SubmissionRequestFinalizer(
     }
 
     private fun deleteRemainingFiles(current: ExtSubmission?, previous: ExtSubmission) {
-        fun deleteFile(index: Int, file: ExtFile) {
-            logger.info { "${previous.accNo} ${previous.owner} Deleting file $index, path='${file.filePath}'" }
-            storageService.deleteSubmissionFile(previous, file)
+        val subFiles = subFilesSet(current)
+        val accNo = previous.accNo
+        val owner = previous.owner
+
+        fun deleteRemainingFiles(allFiles: Sequence<ExtFile>): Sequence<ExtFile> {
+            return allFiles
+                .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode }
+                .onEachIndexed { i, file -> logger.info { "$accNo $owner Deleting file $i, path='${file.filePath}'" } }
         }
 
-        val subFiles = subFilesSet(current)
-        logger.info { "${previous.accNo} ${previous.owner} Started deleting remaining submission files" }
-        serializationService.fileSequence(previous)
-            .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode }
-            .forEachIndexed { index, file -> deleteFile(index, file) }
-        logger.info { "${previous.accNo} ${previous.owner} Finished deleting remaining submission files" }
+        logger.info { "$accNo ${previous.owner} Started deleting remaining submission files" }
+        storageService.deleteSubmissionFiles(previous, ::deleteRemainingFiles)
+        logger.info { "$accNo ${previous.owner} Finished deleting remaining submission files" }
     }
 
     private fun subFilesSet(sub: ExtSubmission?): Set<String> {