From 86f372d98f959abc81867f5b7b48e01cf34d97ae Mon Sep 17 00:00:00 2001 From: Juan Camilo Rada Mesa Date: Thu, 11 May 2023 06:34:09 -0500 Subject: [PATCH 1/2] single commit --- .../filesystem/api/FileStorageService.kt | 2 -- .../filesystem/api/FilesService.kt | 2 -- .../filesystem/fire/FireFilesService.kt | 20 --------------- .../filesystem/nfs/NfsFilesService.kt | 22 +++------------- .../filesystem/service/StorageService.kt | 6 ----- .../filesystem/fire/FireFilesServiceTest.kt | 19 +------------- .../filesystem/nfs/NfsFilesServiceTest.kt | 12 --------- .../common/config/FilePersistenceConfig.kt | 5 +--- .../biostd/common/config/SubmissionConfig.kt | 6 ++--- .../domain/service/SubmissionService.kt | 6 ++--- .../request/SubmissionRequestFinalizer.kt | 25 ++++++++++++------- .../request/SubmissionRequestFinalizerTest.kt | 2 ++ 12 files changed, 30 insertions(+), 97 deletions(-) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt index 00011035c..57afb3d53 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt @@ -9,8 +9,6 @@ interface FileStorageService { fun persistSubmissionFile(sub: ExtSubmission, file: ExtFile): ExtFile - fun deleteSubmissionFiles(sub: ExtSubmission) - fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt index ef3a0a58e..0d9c32534 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt @@ -6,8 +6,6 @@ import ebi.ac.uk.extended.model.ExtSubmission internal interface FilesService { fun persistSubmissionFile(sub: ExtSubmission, file: ExtFile): ExtFile - fun deleteSubmissionFiles(sub: ExtSubmission) - fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt index 5290d7e2c..83c09c6d1 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt @@ -7,16 +7,10 @@ import ebi.ac.uk.extended.model.FireFile import ebi.ac.uk.extended.model.NfsFile import ebi.ac.uk.extended.model.asFireFile import ebi.ac.uk.extended.model.expectedPath -import mu.KotlinLogging -import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence import uk.ac.ebi.fire.client.integration.web.FireClient -private val logger = KotlinLogging.logger {} - class FireFilesService( private val client: FireClient, - private val serializationService: ExtSerializationService, ) : FilesService { /** * Get or persist the given ext file from FIRE. Note that this method assumes that all the fire files belonging to @@ -71,18 +65,4 @@ class FireFilesService( override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { // No need to delete FTP links on FIRE as file deleting complete this } - - override fun deleteSubmissionFiles(sub: ExtSubmission) { - fun deleteFile(index: Int, file: FireFile) { - logger.info { "${sub.accNo} ${sub.owner} Deleting file $index, path='${file.filePath}'" } - deleteSubmissionFile(sub, file) - } - - logger.info { "${sub.accNo} ${sub.owner} Started cleaning submission files for ${sub.accNo}" } - serializationService - .fileSequence(sub) - .filterIsInstance(FireFile::class.java) - .forEachIndexed { index, file -> deleteFile(index, file) } - logger.info { "${sub.accNo} ${sub.owner} Finished cleaning submission files for ${sub.accNo}" } - } } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt index 3aa89568a..b6929694d 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt @@ -62,14 +62,12 @@ class NfsFilesService( return getOrCreateFolder(submissionPath, permissions).toFile() } - override fun deleteSubmissionFiles(sub: ExtSubmission) { - deleteFtpFolder(sub) - deleteSubFolder(sub) - } - override fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) { require(file is NfsFile) { "NfsFilesService should only handle NfsFile" } - FileUtils.deleteFile(folderResolver.getSubFolder(sub.relPath).resolve(file.relPath).toFile()) + + val subDirectory = folderResolver.getSubFolder(sub.relPath) + val subFile = subDirectory.resolve(file.relPath).toFile() + FileUtils.deleteFile(subFile) } override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { @@ -78,16 +76,4 @@ class NfsFilesService( FileUtils.deleteFile(subFolder.resolve(file.relPath).toFile()) logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" } } - - private fun deleteSubFolder(sub: ExtSubmission) { - logger.info { "${sub.accNo} ${sub.owner} Started deleting files of submission ${sub.accNo} on NFS" } - FileUtils.deleteFile(folderResolver.getSubFolder(sub.relPath).toFile()) - logger.info { "${sub.accNo} ${sub.owner} Finished deleting files of submission ${sub.accNo} on NFS" } - } - - private fun deleteFtpFolder(sub: ExtSubmission) { - logger.info { "${sub.accNo} ${sub.owner} Started un-publishing files of submission ${sub.accNo} on NFS" } - FileUtils.deleteFile(folderResolver.getSubmissionFtpFolder(sub.relPath).toFile()) - logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" } - } } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt index 007370e2a..47ab77fb1 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt @@ -42,10 +42,4 @@ class StorageService( FIRE -> fireFilesService.deleteSubmissionFile(sub, file) NFS -> nfsFilesService.deleteSubmissionFile(sub, file) } - - override fun deleteSubmissionFiles(sub: ExtSubmission) = - when (sub.storageMode) { - FIRE -> fireFilesService.deleteSubmissionFiles(sub) - NFS -> nfsFilesService.deleteSubmissionFiles(sub) - } } diff --git a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesServiceTest.kt b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesServiceTest.kt index b0fc7e662..5bcfcc828 100644 --- a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesServiceTest.kt +++ b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesServiceTest.kt @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence import uk.ac.ebi.fire.client.integration.web.FireClient import uk.ac.ebi.fire.client.model.FileSystemEntry import uk.ac.ebi.fire.client.model.FireApiFile @@ -34,7 +33,7 @@ internal class FireFilesServiceTest( @MockK private val submission: ExtSubmission, @MockK private val serializationService: ExtSerializationService, ) { - private val testInstance = FireFilesService(fireClient, serializationService) + private val testInstance = FireFilesService(fireClient) @BeforeEach fun beforeEach() { @@ -142,22 +141,6 @@ internal class FireFilesServiceTest( mockkStatic("uk.ac.ebi.extended.serialization.service.ExtSerializationServiceExtKt") } - @Test - fun `delete submission files`( - @MockK submission: ExtSubmission, - ) { - val file = fireFile(md5 = "md1", firePath = "path_1") - - every { submission.accNo } returns "S-BSST1" - every { submission.owner } returns "user@mail.org" - every { serializationService.fileSequence(submission) } returns sequenceOf(file) - every { fireClient.delete(file.fireId) } answers { nothing } - - testInstance.deleteSubmissionFiles(submission) - - verify(exactly = 1) { fireClient.delete(file.fireId) } - } - @Test fun `delete submission file`( @MockK submission: ExtSubmission, diff --git a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt index a27a71c14..1a2aad9a7 100644 --- a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt +++ b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt @@ -59,18 +59,6 @@ class NfsFilesServiceTest( assertThat(Files.exists(Paths.get("${subFolder.absolutePath}/${sub.relPath}/${fireFile.relPath}"))).isTrue() } - @Test - fun `delete submission files`() { - val subFolder = subFolder.createDirectory("S-BSST1") - val ftpFolder = ftpFolder.createDirectory("S-BSST1") - val sub = basicExtSubmission.copy(relPath = "S-BSST1") - - testInstance.deleteSubmissionFiles(sub) - - assertThat(Files.exists(subFolder.toPath())).isFalse() - assertThat(Files.exists(ftpFolder.toPath())).isFalse() - } - @Test fun `delete ftp links`() { val ftpFolder = ftpFolder.createDirectory("S-BSST2") diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt index 14a920481..35edb0c3c 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt @@ -17,7 +17,6 @@ import ebi.ac.uk.paths.SubmissionFolderResolver import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import -import uk.ac.ebi.extended.serialization.service.ExtSerializationService import uk.ac.ebi.fire.client.integration.web.FireClient import java.io.File @@ -48,9 +47,7 @@ class FilePersistenceConfig( fun fireFtpService(): FireFtpService = FireFtpService(fireClient) @Bean - fun fireFileService( - serializationService: ExtSerializationService, - ): FireFilesService = FireFilesService(fireClient, serializationService) + fun fireFileService(): FireFilesService = FireFilesService(fireClient) @Bean fun pageTabService( diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt index 5471e1fa5..0f0194f12 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt @@ -9,7 +9,6 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionDraftPersistenceSer import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService -import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ac.uk.ebi.biostd.stats.domain.service.SubmissionStatsService import ac.uk.ebi.biostd.stats.web.handlers.StatsFileHandler import ac.uk.ebi.biostd.submission.domain.helpers.CollectionService @@ -25,6 +24,7 @@ import ac.uk.ebi.biostd.submission.domain.service.SubmissionStagesHandler import ac.uk.ebi.biostd.submission.service.FileSourcesService import ac.uk.ebi.biostd.submission.submitter.ExtSubmissionSubmitter import ac.uk.ebi.biostd.submission.submitter.SubmissionSubmitter +import ac.uk.ebi.biostd.submission.submitter.request.SubmissionRequestFinalizer import ac.uk.ebi.biostd.submission.web.handlers.SubmissionsWebHandler import ac.uk.ebi.biostd.submission.web.handlers.SubmitRequestBuilder import ac.uk.ebi.biostd.submission.web.handlers.SubmitWebHandler @@ -64,15 +64,15 @@ class SubmissionConfig( extSubmissionSubmitter: ExtSubmissionSubmitter, submissionSubmitter: SubmissionSubmitter, eventsPublisherService: EventsPublisherService, - fileStorageService: FileStorageService, + submissionRequestFinalizer: SubmissionRequestFinalizer, ): SubmissionService = SubmissionService( submissionPersistenceQueryService, userPrivilegeService, extSubmissionSubmitter, submissionSubmitter, eventsPublisherService, - fileStorageService, submissionPersistenceService, + submissionRequestFinalizer ) @Bean diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt index cea2a33ec..1a684c941 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt @@ -2,13 +2,13 @@ package ac.uk.ebi.biostd.submission.domain.service import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService -import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ac.uk.ebi.biostd.submission.exceptions.UserCanNotDelete import ac.uk.ebi.biostd.submission.exceptions.UserCanNotRelease import ac.uk.ebi.biostd.submission.model.ReleaseRequest import ac.uk.ebi.biostd.submission.model.SubmitRequest import ac.uk.ebi.biostd.submission.submitter.ExtSubmissionSubmitter import ac.uk.ebi.biostd.submission.submitter.SubmissionSubmitter +import ac.uk.ebi.biostd.submission.submitter.request.SubmissionRequestFinalizer import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.security.integration.components.IUserPrivilegesService import ebi.ac.uk.security.integration.model.api.SecurityUser @@ -24,8 +24,8 @@ class SubmissionService( private val extSubmissionSubmitter: ExtSubmissionSubmitter, private val submissionSubmitter: SubmissionSubmitter, private val eventsPublisherService: EventsPublisherService, - private val fileStorageService: FileStorageService, private val submissionPersistenceService: SubmissionPersistenceService, + private val submissionRequestFinalizer: SubmissionRequestFinalizer, ) { fun submit(rqt: SubmitRequest): ExtSubmission { logger.info { "${rqt.accNo} ${rqt.owner} Received sync submit request with draft key '${rqt.draftKey}'" } @@ -40,7 +40,7 @@ class SubmissionService( fun deleteSubmission(accNo: String, user: SecurityUser) { require(userPrivilegesService.canDelete(user.email, accNo)) { throw UserCanNotDelete(accNo, user.email) } - fileStorageService.deleteSubmissionFiles(queryService.getExtByAccNo(accNo, true)) + submissionRequestFinalizer.deleteSubmissionFiles(queryService.getExtByAccNo(accNo, true)) submissionPersistenceService.expireSubmission(accNo) eventsPublisherService.submissionsRefresh(accNo, user.email) } diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt index b2ebcf3c0..2c8e3fb2a 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt @@ -30,22 +30,29 @@ class SubmissionRequestFinalizer( return sub } - private fun deleteRemainingFiles(sub: ExtSubmission, previous: ExtSubmission) { + fun deleteSubmissionFiles(submission: ExtSubmission) { + deleteRemainingFiles(null, submission) + } + + private fun deleteRemainingFiles(current: ExtSubmission?, previous: ExtSubmission) { fun deleteFile(index: Int, file: ExtFile) { logger.info { "${previous.accNo} ${previous.owner} Deleting file $index, path='${file.filePath}'" } storageService.deleteSubmissionFile(previous, file) + storageService.deleteFtpFile(previous, file) } - val subFiles = subFilesSet(sub) - logger.info { "${sub.accNo} ${sub.owner} Started deleting remaining submission files" } + val subFiles = subFilesSet(current) + logger.info { "${previous.accNo} ${previous.owner} Started deleting remaining submission files" } serializationService.fileSequence(previous) - .filter { subFiles.contains(it.filePath).not() || it.storageMode != sub.storageMode } + .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode } .forEachIndexed { index, file -> deleteFile(index, file) } - logger.info { "${sub.accNo} ${sub.owner} Finished deleting remaining submission files" } + logger.info { "${previous.accNo} ${previous.owner} Finished deleting remaining submission files" } } - private fun subFilesSet(sub: ExtSubmission) = - serializationService.fileSequence(sub) - .map { it.filePath } - .toSet() + private fun subFilesSet(sub: ExtSubmission?): Set { + return when (sub) { + null -> emptySet() + else -> serializationService.fileSequence(sub).mapTo(mutableSetOf()) { it.filePath } + } + } } diff --git a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt index 3d808dee9..73baa8714 100644 --- a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt +++ b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt @@ -81,6 +81,7 @@ class SubmissionRequestFinalizerTest( every { requestService.getPersistedRequest("S-BSST1", 2) } returns persistedRequest every { serializationService.fileSequence(previous) } returns sequenceOf(previousFile) every { storageService.deleteSubmissionFile(previous, previousFile) } answers { nothing } + every { storageService.deleteFtpFile(previous, previousFile) } answers { nothing } every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) testInstance.finalizeRequest("S-BSST1", 2) @@ -111,6 +112,7 @@ class SubmissionRequestFinalizerTest( every { serializationService.fileSequence(previous) } returns sequenceOf(subFile) every { queryService.findLatestInactiveByAccNo("S-BSST1", true) } returns previous every { storageService.deleteSubmissionFile(previous, subFile) } answers { nothing } + every { storageService.deleteFtpFile(previous, subFile) } answers { nothing } every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) testInstance.finalizeRequest("S-BSST1", 2) From 76603f4c3599394deec10f718009b4f999ce275d Mon Sep 17 00:00:00 2001 From: Juan Camilo Rada Mesa Date: Tue, 16 May 2023 07:48:19 -0500 Subject: [PATCH 2/2] code review changes --- .../filesystem/api/FileStorageService.kt | 2 +- .../filesystem/service/StorageService.kt | 26 +++++++++++++------ .../common/config/FilePersistenceConfig.kt | 5 +++- .../biostd/common/config/SubmissionConfig.kt | 6 ++--- .../domain/service/SubmissionService.kt | 6 ++--- .../src/main/resources/application.yml | 5 ++++ .../request/SubmissionRequestCleaner.kt | 1 - .../request/SubmissionRequestFinalizer.kt | 5 ---- .../request/SubmissionRequestCleanerTest.kt | 3 --- .../request/SubmissionRequestFinalizerTest.kt | 2 -- 10 files changed, 34 insertions(+), 27 deletions(-) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt index 57afb3d53..af59f9c5a 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt @@ -11,5 +11,5 @@ interface FileStorageService { fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) - fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) + fun deleteSubmissionFiles(sub: ExtSubmission) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt index 47ab77fb1..03478c67f 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt @@ -10,6 +10,11 @@ import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.StorageMode import ebi.ac.uk.extended.model.StorageMode.FIRE import ebi.ac.uk.extended.model.StorageMode.NFS +import mu.KotlinLogging +import uk.ac.ebi.extended.serialization.service.ExtSerializationService +import uk.ac.ebi.extended.serialization.service.fileSequence + +private val logger = KotlinLogging.logger {} @Suppress("LongParameterList") class StorageService( @@ -17,6 +22,7 @@ class StorageService( private val fireFilesService: FireFilesService, private val nfsFtpService: NfsFtpService, private val nfsFilesService: NfsFilesService, + private val serializationService: ExtSerializationService, ) : FileStorageService { override fun persistSubmissionFile(sub: ExtSubmission, file: ExtFile): ExtFile = when (sub.storageMode) { @@ -31,15 +37,19 @@ class StorageService( } } - override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) = - when (sub.storageMode) { - FIRE -> fireFilesService.deleteFtpFile(sub, file) - NFS -> nfsFilesService.deleteFtpFile(sub, file) + override fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) = when (sub.storageMode) { + FIRE -> { + fireFilesService.deleteSubmissionFile(sub, file) + fireFilesService.deleteFtpFile(sub, file) } - override fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) = - when (sub.storageMode) { - FIRE -> fireFilesService.deleteSubmissionFile(sub, file) - NFS -> nfsFilesService.deleteSubmissionFile(sub, file) + NFS -> { + nfsFilesService.deleteSubmissionFile(sub, file) + nfsFilesService.deleteFtpFile(sub, file) } + } + + override fun deleteSubmissionFiles(sub: ExtSubmission) { + serializationService.fileSequence(sub).forEach { file -> deleteSubmissionFile(sub, file) } + } } diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt index 35edb0c3c..7f2fd2404 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/FilePersistenceConfig.kt @@ -17,6 +17,7 @@ import ebi.ac.uk.paths.SubmissionFolderResolver import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Import +import uk.ac.ebi.extended.serialization.service.ExtSerializationService import uk.ac.ebi.fire.client.integration.web.FireClient import java.io.File @@ -35,7 +36,9 @@ class FilePersistenceConfig( fireFilesService: FireFilesService, nfsFtpService: NfsFtpService, nfsFilesService: NfsFilesService, - ): FileStorageService = StorageService(fireFtpService, fireFilesService, nfsFtpService, nfsFilesService) + extSerializationService: ExtSerializationService, + ): FileStorageService = + StorageService(fireFtpService, fireFilesService, nfsFtpService, nfsFilesService, extSerializationService) @Bean fun nfsFtpService(): NfsFtpService = NfsFtpService(folderResolver) diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt index 0f0194f12..bf238f73b 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt @@ -9,6 +9,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionDraftPersistenceSer import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService +import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ac.uk.ebi.biostd.stats.domain.service.SubmissionStatsService import ac.uk.ebi.biostd.stats.web.handlers.StatsFileHandler import ac.uk.ebi.biostd.submission.domain.helpers.CollectionService @@ -24,7 +25,6 @@ import ac.uk.ebi.biostd.submission.domain.service.SubmissionStagesHandler import ac.uk.ebi.biostd.submission.service.FileSourcesService import ac.uk.ebi.biostd.submission.submitter.ExtSubmissionSubmitter import ac.uk.ebi.biostd.submission.submitter.SubmissionSubmitter -import ac.uk.ebi.biostd.submission.submitter.request.SubmissionRequestFinalizer import ac.uk.ebi.biostd.submission.web.handlers.SubmissionsWebHandler import ac.uk.ebi.biostd.submission.web.handlers.SubmitRequestBuilder import ac.uk.ebi.biostd.submission.web.handlers.SubmitWebHandler @@ -64,7 +64,7 @@ class SubmissionConfig( extSubmissionSubmitter: ExtSubmissionSubmitter, submissionSubmitter: SubmissionSubmitter, eventsPublisherService: EventsPublisherService, - submissionRequestFinalizer: SubmissionRequestFinalizer, + fileStorageService: FileStorageService, ): SubmissionService = SubmissionService( submissionPersistenceQueryService, userPrivilegeService, @@ -72,7 +72,7 @@ class SubmissionConfig( submissionSubmitter, eventsPublisherService, submissionPersistenceService, - submissionRequestFinalizer + fileStorageService ) @Bean diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt index 1a684c941..150acfc3e 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt @@ -2,13 +2,13 @@ package ac.uk.ebi.biostd.submission.domain.service import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService +import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ac.uk.ebi.biostd.submission.exceptions.UserCanNotDelete import ac.uk.ebi.biostd.submission.exceptions.UserCanNotRelease import ac.uk.ebi.biostd.submission.model.ReleaseRequest import ac.uk.ebi.biostd.submission.model.SubmitRequest import ac.uk.ebi.biostd.submission.submitter.ExtSubmissionSubmitter import ac.uk.ebi.biostd.submission.submitter.SubmissionSubmitter -import ac.uk.ebi.biostd.submission.submitter.request.SubmissionRequestFinalizer import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.security.integration.components.IUserPrivilegesService import ebi.ac.uk.security.integration.model.api.SecurityUser @@ -25,7 +25,7 @@ class SubmissionService( private val submissionSubmitter: SubmissionSubmitter, private val eventsPublisherService: EventsPublisherService, private val submissionPersistenceService: SubmissionPersistenceService, - private val submissionRequestFinalizer: SubmissionRequestFinalizer, + private val fileStorageService: FileStorageService, ) { fun submit(rqt: SubmitRequest): ExtSubmission { logger.info { "${rqt.accNo} ${rqt.owner} Received sync submit request with draft key '${rqt.draftKey}'" } @@ -40,7 +40,7 @@ class SubmissionService( fun deleteSubmission(accNo: String, user: SecurityUser) { require(userPrivilegesService.canDelete(user.email, accNo)) { throw UserCanNotDelete(accNo, user.email) } - submissionRequestFinalizer.deleteSubmissionFiles(queryService.getExtByAccNo(accNo, true)) + fileStorageService.deleteSubmissionFiles(queryService.getExtByAccNo(accNo, true)) submissionPersistenceService.expireSubmission(accNo) eventsPublisherService.submissionsRefresh(accNo, user.email) } diff --git a/submission/submission-webapp/src/main/resources/application.yml b/submission/submission-webapp/src/main/resources/application.yml index 957ed2e08..2d431f017 100644 --- a/submission/submission-webapp/src/main/resources/application.yml +++ b/submission/submission-webapp/src/main/resources/application.yml @@ -72,6 +72,11 @@ app: username: # FIRE user password: # FIRE password version: v1.1 + s3AccessKey: # FIRE s3 access key + s3SecretKey: # FIRE s3 access key + s3region: anyRegion # Fire s3 region + s3endpoint: # Fire s3 region + s3bucket: # Fire s3 region retry: maxAttempts: 20 initialInterval: 100 diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt index ad8a95c52..fbf2d06de 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt @@ -40,7 +40,6 @@ class SubmissionRequestCleaner( fun deleteFile(index: Int, file: ExtFile) { logger.info { "${current.accNo} ${current.owner} Deleting file $index, path='${file.filePath}'" } storageService.deleteSubmissionFile(current, file) - storageService.deleteFtpFile(current, file) } fun shouldDelete(newFiles: Map, existing: ExtFile): Boolean = diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt index 2c8e3fb2a..b9bf22023 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt @@ -30,15 +30,10 @@ class SubmissionRequestFinalizer( return sub } - fun deleteSubmissionFiles(submission: ExtSubmission) { - deleteRemainingFiles(null, submission) - } - private fun deleteRemainingFiles(current: ExtSubmission?, previous: ExtSubmission) { fun deleteFile(index: Int, file: ExtFile) { logger.info { "${previous.accNo} ${previous.owner} Deleting file $index, path='${file.filePath}'" } storageService.deleteSubmissionFile(previous, file) - storageService.deleteFtpFile(previous, file) } val subFiles = subFilesSet(current) diff --git a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt index f080f0cc2..89704efba 100644 --- a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt +++ b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt @@ -64,7 +64,6 @@ class SubmissionRequestCleanerTest( verify(exactly = 1) { requestService.saveSubmissionRequest(cleanedRequest) } verify(exactly = 0) { - storageService.deleteFtpFile(any(), any()) storageService.deleteSubmissionFile(any(), any()) } } @@ -89,7 +88,6 @@ class SubmissionRequestCleanerTest( every { loadedRequest.submission } returns new - every { storageService.deleteFtpFile(current, currentFile) } answers { nothing } every { loadedRequest.withNewStatus(CLEANED) } returns cleanedRequest every { queryService.findExtByAccNo("S-BSST1", true) } returns current every { requestService.getLoadedRequest("S-BSST1", 2) } returns loadedRequest @@ -101,7 +99,6 @@ class SubmissionRequestCleanerTest( testInstance.cleanCurrentVersion("S-BSST1", 2) verify(exactly = 1) { - storageService.deleteFtpFile(current, currentFile) requestService.saveSubmissionRequest(cleanedRequest) storageService.deleteSubmissionFile(current, currentFile) } diff --git a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt index 73baa8714..3d808dee9 100644 --- a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt +++ b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizerTest.kt @@ -81,7 +81,6 @@ class SubmissionRequestFinalizerTest( every { requestService.getPersistedRequest("S-BSST1", 2) } returns persistedRequest every { serializationService.fileSequence(previous) } returns sequenceOf(previousFile) every { storageService.deleteSubmissionFile(previous, previousFile) } answers { nothing } - every { storageService.deleteFtpFile(previous, previousFile) } answers { nothing } every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) testInstance.finalizeRequest("S-BSST1", 2) @@ -112,7 +111,6 @@ class SubmissionRequestFinalizerTest( every { serializationService.fileSequence(previous) } returns sequenceOf(subFile) every { queryService.findLatestInactiveByAccNo("S-BSST1", true) } returns previous every { storageService.deleteSubmissionFile(previous, subFile) } answers { nothing } - every { storageService.deleteFtpFile(previous, subFile) } answers { nothing } every { requestService.saveSubmissionRequest(processedRequest) } returns ("S-BSST1" to 1) testInstance.finalizeRequest("S-BSST1", 2)