diff --git a/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt b/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt index 9c939db79..960cfd806 100644 --- a/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt +++ b/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt @@ -6,6 +6,7 @@ import ebi.ac.uk.io.FileUtilsHelper.createFolderHardLinks import ebi.ac.uk.io.FileUtilsHelper.createFolderIfNotExist import ebi.ac.uk.io.FileUtilsHelper.createParentDirectories import ebi.ac.uk.io.FileUtilsHelper.createSymLink +import ebi.ac.uk.io.ext.isEmpty import ebi.ac.uk.io.ext.notExist import ebi.ac.uk.io.ext.size import org.apache.commons.codec.digest.DigestUtils @@ -72,6 +73,15 @@ object FileUtils { } } + fun deleteEmptyDirectories(dir: File) { + dir + .walkBottomUp() + .asSequence() + .takeWhile { it != dir } + .forEach { deleteEmptyDirectories(it) } + if (dir.isDirectory && dir.isEmpty()) Files.delete(dir.toPath()) + } + fun moveFile( source: File, target: File, diff --git a/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt b/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt index 90b98abe7..e899f9880 100644 --- a/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt +++ b/commons/commons-util/src/main/kotlin/ebi/ac/uk/io/ext/FileExtensions.kt @@ -14,6 +14,8 @@ fun File.notExist() = Files.exists(toPath()).not() fun File.listFilesOrEmpty(): List = FileUtils.listFiles(this) +fun File.isEmpty(): Boolean = Files.newDirectoryStream(this.toPath()).use { return it.iterator().hasNext().not() } + fun File.allSubFiles(): List = FileUtils.listAllFiles(this) fun File.size(calculateDirectories: Boolean = true) = FileUtils.size(this, calculateDirectories) diff --git a/commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt b/commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt index 7eb2948f5..24f542fe4 100644 --- a/commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt +++ b/commons/commons-util/src/test/kotlin/ebi/ac/uk/io/FileUtilsTest.kt @@ -267,6 +267,40 @@ internal class FileUtilsTest(private val temporaryFolder: TemporaryFolder) { } } + @Nested + inner class DeleteEmptyFolders { + @Test + fun whenEmpty() { + val main = temporaryFolder.createDirectory("main") + val sub1 = main.createDirectory("sub1") + val sub2 = main.createDirectory("sub2") + sub1.createDirectory("sub1Sub1") + sub1.createDirectory("sub1Sub2") + sub2.createDirectory("sub2Sub1") + + FileUtils.deleteEmptyDirectories(main) + + assertThat(main).doesNotExist() + } + + @Test + fun whenNotEmpty() { + val main = temporaryFolder.createDirectory("main") + val sub1 = main.createDirectory("sub1") + val sub2 = main.createDirectory("sub2") + val sub1Sub1 = sub1.createDirectory("sub1Sub1") + val file = sub1Sub1.createFile("one.txt", "text") + + FileUtils.deleteEmptyDirectories(main) + + assertThat(file).exists() + assertThat(sub1Sub1).exists() + assertThat(sub2).doesNotExist() + assertThat(sub1).exists() + assertThat(main).exists() + } + } + @Nested inner class Utilities { @Test diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt index af59f9c5a..5a961e822 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt @@ -11,5 +11,8 @@ interface FileStorageService { fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) - fun deleteSubmissionFiles(sub: ExtSubmission) + fun deleteSubmissionFiles( + sub: ExtSubmission, + process: (Sequence) -> Sequence = { it }, + ) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt index 0d9c32534..9bd35d67a 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt @@ -9,4 +9,6 @@ internal interface FilesService { fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) + + fun deleteEmptyFolders(current: ExtSubmission) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt index 83c09c6d1..374797f83 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt @@ -65,4 +65,8 @@ class FireFilesService( override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { // No need to delete FTP links on FIRE as file deleting complete this } + + override fun deleteEmptyFolders(current: ExtSubmission) { + // No need to delete FIRE empty bucket as they only exists as files are in them + } } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt index b6929694d..7250307f5 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt @@ -76,4 +76,9 @@ class NfsFilesService( FileUtils.deleteFile(subFolder.resolve(file.relPath).toFile()) logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" } } + + override fun deleteEmptyFolders(current: ExtSubmission) { + val subFolder = folderResolver.getSubFolder(current.relPath) + FileUtils.deleteEmptyDirectories(subFolder.toFile()) + } } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt index 03478c67f..47846f1f6 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt @@ -49,7 +49,16 @@ class StorageService( } } - override fun deleteSubmissionFiles(sub: ExtSubmission) { - serializationService.fileSequence(sub).forEach { file -> deleteSubmissionFile(sub, file) } + override fun deleteSubmissionFiles( + sub: ExtSubmission, + process: (Sequence) -> Sequence, + ) { + process(serializationService.fileSequence(sub)).forEach { file -> deleteSubmissionFile(sub, file) } + deleteEmptyFolders(sub) + } + + private fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) { + FIRE -> fireFilesService.deleteEmptyFolders(sub) + NFS -> nfsFilesService.deleteEmptyFolders(sub) } } diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt index b9bf22023..d7c5dc434 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt @@ -31,17 +31,19 @@ class SubmissionRequestFinalizer( } private fun deleteRemainingFiles(current: ExtSubmission?, previous: ExtSubmission) { - fun deleteFile(index: Int, file: ExtFile) { - logger.info { "${previous.accNo} ${previous.owner} Deleting file $index, path='${file.filePath}'" } - storageService.deleteSubmissionFile(previous, file) + val subFiles = subFilesSet(current) + val accNo = previous.accNo + val owner = previous.owner + + fun deleteRemainingFiles(allFiles: Sequence): Sequence { + return allFiles + .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode } + .onEachIndexed { i, file -> logger.info { "$accNo $owner Deleting file $i, path='${file.filePath}'" } } } - val subFiles = subFilesSet(current) - logger.info { "${previous.accNo} ${previous.owner} Started deleting remaining submission files" } - serializationService.fileSequence(previous) - .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode } - .forEachIndexed { index, file -> deleteFile(index, file) } - logger.info { "${previous.accNo} ${previous.owner} Finished deleting remaining submission files" } + logger.info { "$accNo ${previous.owner} Started deleting remaining submission files" } + storageService.deleteSubmissionFiles(previous, ::deleteRemainingFiles) + logger.info { "$accNo ${previous.owner} Finished deleting remaining submission files" } } private fun subFilesSet(sub: ExtSubmission?): Set { diff --git a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt deleted file mode 100644 index 3d808dee9..000000000 --- a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt +++ /dev/null @@ -1,123 +0,0 @@ -package ac.uk.ebi.biostd.submission.submitter.request - -import ac.uk.ebi.biostd.persistence.common.model.RequestStatus.PROCESSED -import ac.uk.ebi.biostd.persistence.common.model.SubmissionRequest -import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService -import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService -import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService -import ebi.ac.uk.extended.model.ExtSubmission -import ebi.ac.uk.extended.model.FireFile -import ebi.ac.uk.extended.model.NfsFile -import ebi.ac.uk.extended.model.StorageMode.FIRE -import ebi.ac.uk.extended.model.StorageMode.NFS -import io.mockk.clearAllMocks -import io.mockk.every -import io.mockk.impl.annotations.MockK -import io.mockk.junit5.MockKExtension -import io.mockk.mockkStatic -import io.mockk.verify -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence - -@ExtendWith(MockKExtension::class) -class SubmissionRequestFinalizerTest( - @MockK private val storageService: FileStorageService, - @MockK private val serializationService: ExtSerializationService, - @MockK private val queryService: SubmissionPersistenceQueryService, - @MockK private val requestService: SubmissionRequestPersistenceService, -) { - private val testInstance = SubmissionRequestFinalizer( - storageService, - serializationService, - queryService, - requestService, - ) - - @BeforeEach - fun beforeEach() { - mockkStatic("uk.ac.ebi.extended.serialization.service.ExtSerializationServiceExtKt") - } - - @AfterEach - fun afterEach() = clearAllMocks() - - @Test - fun `delete remaining without previous version`( - @MockK new: ExtSubmission, - @MockK persistedRequest: SubmissionRequest, - @MockK processedRequest: SubmissionRequest, - ) { - every { queryService.getExtByAccNo("S-BSST1", true) } returns new - every { persistedRequest.withNewStatus(PROCESSED) } returns processedRequest - every { queryService.findLatestInactiveByAccNo("S-BSST1", true) } returns null - every { requestService.getPersistedRequest("S-BSST1", 1) } returns persistedRequest - every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) - - testInstance.finalizeRequest("S-BSST1", 1) - - verify(exactly = 1) { requestService.saveSubmissionRequest(processedRequest) } - verify(exactly = 0) { storageService.deleteSubmissionFile(any(), any()) } - } - - @Test - fun `delete remaining from previous version different storage mode`( - @MockK new: ExtSubmission, - @MockK previousFile: NfsFile, - @MockK previous: ExtSubmission, - @MockK persistedRequest: SubmissionRequest, - @MockK processedRequest: SubmissionRequest, - ) { - every { new.storageMode } returns FIRE - every { previous.storageMode } returns NFS - every { previousFile.filePath } returns "a/b/text.txt" - every { queryService.getExtByAccNo("S-BSST1", true) } returns new - every { serializationService.fileSequence(new) } returns emptySequence() - every { persistedRequest.withNewStatus(PROCESSED) } returns processedRequest - every { queryService.findLatestInactiveByAccNo("S-BSST1", true) } returns previous - every { requestService.getPersistedRequest("S-BSST1", 2) } returns persistedRequest - every { serializationService.fileSequence(previous) } returns sequenceOf(previousFile) - every { storageService.deleteSubmissionFile(previous, previousFile) } answers { nothing } - every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) - - testInstance.finalizeRequest("S-BSST1", 2) - - verify(exactly = 1) { - requestService.saveSubmissionRequest(processedRequest) - storageService.deleteSubmissionFile(previous, previousFile) - } - } - - @Test - fun `delete remaining from previous version`( - @MockK subFile: FireFile, - @MockK new: ExtSubmission, - @MockK previous: ExtSubmission, - @MockK persistedRequest: SubmissionRequest, - @MockK processedRequest: SubmissionRequest, - ) { - every { new.storageMode } returns FIRE - every { previous.accNo } returns "S-BSST1" - every { previous.storageMode } returns FIRE - every { previous.owner } returns "owner@mail.org" - every { subFile.filePath } returns "a/b/text.txt" - every { queryService.getExtByAccNo("S-BSST1", true) } returns new - every { serializationService.fileSequence(new) } returns emptySequence() - every { persistedRequest.withNewStatus(PROCESSED) } returns processedRequest - every { requestService.getPersistedRequest("S-BSST1", 2) } returns persistedRequest - every { serializationService.fileSequence(previous) } returns sequenceOf(subFile) - every { queryService.findLatestInactiveByAccNo("S-BSST1", true) } returns previous - every { storageService.deleteSubmissionFile(previous, subFile) } answers { nothing } - every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) - - testInstance.finalizeRequest("S-BSST1", 2) - - verify(exactly = 1) { - storageService.deleteSubmissionFile(previous, subFile) - requestService.saveSubmissionRequest(processedRequest) - } - } -}