From a20c49ec0cd7167860d983a90d809225b7582b49 Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 3 Feb 2023 00:33:08 -0800 Subject: [PATCH] Pivotal ID # 184364448: Previos version file check taking longer than expected (#678) https://www.pivotaltracker.com/story/show/184364448 - updated api to avoid re query submission on each call --- .../common/service/SubmissionOperations.kt | 4 +- .../data/FileListDocFileDocDataRepository.kt | 2 +- .../doc/db/repositories/Repositories.kt | 2 +- .../doc/integration/MongoDbReposConfig.kt | 10 +-- .../doc/integration/MongoDbServicesConfig.kt | 3 +- .../doc/migrations/DatabaseChangeLog.kt | 17 +++++ .../SubmissionMongoPersistenceQueryService.kt | 24 ++++--- .../FileListDocFileRepositoryTest.kt | 64 +++++++++++++++++++ .../SubmissionMongoQueryServiceTest.kt | 55 ++++++++++------ .../submit/SubmissionFileSourceTest.kt | 14 ++-- .../biostd/common/config/SubmissionConfig.kt | 3 +- .../service/ExtSubmissionQueryService.kt | 10 +-- .../src/main/resources/application.yml | 2 + ...xtSubmissionPersistenceQueryServiceTest.kt | 4 +- .../submission/helpers/FilesSourceFactory.kt | 3 +- .../helpers/SubmissionFilesSource.kt | 6 +- .../helpers/FilesSourceFactoryTest.kt | 5 +- 17 files changed, 164 insertions(+), 64 deletions(-) create mode 100644 submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/FileListDocFileRepositoryTest.kt diff --git a/submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/common/service/SubmissionOperations.kt b/submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/common/service/SubmissionOperations.kt index 7244bb9ad1..90491aa998 100644 --- a/submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/common/service/SubmissionOperations.kt +++ b/submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/common/service/SubmissionOperations.kt @@ -49,9 +49,9 @@ interface SubmissionPersistenceQueryService { **/ fun getSubmissionsByUser(owner: String, filter: SubmissionFilter): List - fun getReferencedFiles(accNo: String, fileListName: String): List + fun getReferencedFiles(sub: ExtSubmission, fileListName: String): List - fun findReferencedFile(accNo: String, version: Int, path: String): ExtFile? + fun findReferencedFile(sub: ExtSubmission, path: String): ExtFile? } @Suppress("TooManyFunctions") diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt index 9c0336f0c3..5d1ef2e853 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt @@ -3,5 +3,5 @@ package ac.uk.ebi.biostd.persistence.doc.db.data import ac.uk.ebi.biostd.persistence.doc.db.repositories.FileListDocFileRepository class FileListDocFileDocDataRepository( - private val fileListDocFileRepository: FileListDocFileRepository + private val fileListDocFileRepository: FileListDocFileRepository, ) : FileListDocFileRepository by fileListDocFileRepository diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/Repositories.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/Repositories.kt index cdfbcee9f5..608201c7eb 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/Repositories.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/Repositories.kt @@ -104,7 +104,7 @@ interface FileListDocFileRepository : MongoRepository fileListName: String, ): List - @Query("{ 'submissionAccNo': ?0, 'submissionVersion': ?1, 'file.filePath': ?2}") + @Query("{ 'submissionAccNo': ?0, 'submissionVersion': ?1, 'file.filePath': ?2}") fun findBySubmissionAccNoAndSubmissionVersionAndFilePath( accNo: String, version: Int, diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbReposConfig.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbReposConfig.kt index 3593a3efc4..e5f36b970e 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbReposConfig.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbReposConfig.kt @@ -23,20 +23,20 @@ class MongoDbReposConfig { @Bean internal fun submissionDocDataRepository( mongoTemplate: MongoTemplate, - submissionMongoRepository: SubmissionMongoRepository + submissionMongoRepository: SubmissionMongoRepository, ): SubmissionDocDataRepository = SubmissionDocDataRepository(submissionMongoRepository, mongoTemplate) @Bean internal fun submissionStatsDataRepository( mongoTemplate: MongoTemplate, - submissionStatsRepository: SubmissionStatsRepository + submissionStatsRepository: SubmissionStatsRepository, ): SubmissionStatsDataRepository = SubmissionStatsDataRepository(mongoTemplate, submissionStatsRepository) @Bean internal fun submissionRequestDocDataRepository( mongoTemplate: MongoTemplate, extSerializationService: ExtSerializationService, - submissionRequestRepository: SubmissionRequestRepository + submissionRequestRepository: SubmissionRequestRepository, ): SubmissionRequestDocDataRepository = SubmissionRequestDocDataRepository( mongoTemplate, extSerializationService, @@ -46,11 +46,11 @@ class MongoDbReposConfig { @Bean internal fun submissionDraftDocDataRepository( mongoTemplate: MongoTemplate, - submissionDraftRepository: SubmissionDraftRepository + submissionDraftRepository: SubmissionDraftRepository, ): SubmissionDraftDocDataRepository = SubmissionDraftDocDataRepository(submissionDraftRepository, mongoTemplate) @Bean internal fun fileListDocFileDocDataRepository( - fileListDocFileRepository: FileListDocFileRepository + fileListDocFileRepository: FileListDocFileRepository, ): FileListDocFileDocDataRepository = FileListDocFileDocDataRepository(fileListDocFileRepository) } diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbServicesConfig.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbServicesConfig.kt index 3c50dabd22..7f622fce39 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbServicesConfig.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/integration/MongoDbServicesConfig.kt @@ -6,6 +6,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionDraftPersistenceSer import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestFilesPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService +import ac.uk.ebi.biostd.persistence.doc.db.data.FileListDocFileDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionDraftDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionRequestDocDataRepository @@ -38,7 +39,7 @@ import uk.ac.ebi.serialization.common.FilesResolver class MongoDbServicesConfig { @Bean internal fun submissionQueryService( - fileListDocFileRepository: FileListDocFileRepository, + fileListDocFileRepository: FileListDocFileDocDataRepository, submissionDocDataRepository: SubmissionDocDataRepository, submissionRequestDocDataRepository: SubmissionRequestDocDataRepository, serializationService: ExtSerializationService, diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/migrations/DatabaseChangeLog.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/migrations/DatabaseChangeLog.kt index 671763e870..c01a179f98 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/migrations/DatabaseChangeLog.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/migrations/DatabaseChangeLog.kt @@ -1,6 +1,7 @@ package ac.uk.ebi.biostd.persistence.doc.migrations import ac.uk.ebi.biostd.persistence.doc.commons.ensureExists +import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocFileFields import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSectionFields.SEC_TYPE import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionFields.SUB import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionFields.SUB_ACC_NO @@ -18,6 +19,7 @@ import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionReques import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionRequestFileFields.RQT_FILE_PATH import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionRequestFileFields.RQT_FILE_SUB_ACC_NO import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.DocSubmissionRequestFileFields.RQT_FILE_SUB_VERSION +import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.FileListDocFileFields.FILE_LIST_DOC_FILE_FILE import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.FileListDocFileFields.FILE_LIST_DOC_FILE_FILE_LIST_NAME import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.FileListDocFileFields.FILE_LIST_DOC_FILE_INDEX import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.FileListDocFileFields.FILE_LIST_DOC_FILE_SUBMISSION_ACC_NO @@ -187,3 +189,18 @@ class ChangeLog007 { } } } + +@ChangeLog +class ChangeLog008 { + @ChangeSet(order = "008", id = "File List files index", author = "System") + fun changeSet007(template: MongockTemplate) { + template.indexOps(FileListDocFile::class.java).apply { + ensureIndex( + Index() + .on(FILE_LIST_DOC_FILE_SUBMISSION_ACC_NO, ASC) + .on(FILE_LIST_DOC_FILE_SUBMISSION_VERSION, ASC) + .on("$FILE_LIST_DOC_FILE_FILE.${DocFileFields.FILE_DOC_FILEPATH}", ASC) + ) + } + } +} diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoPersistenceQueryService.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoPersistenceQueryService.kt index 27e21c4b4f..c55c5710b8 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoPersistenceQueryService.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoPersistenceQueryService.kt @@ -3,9 +3,9 @@ package ac.uk.ebi.biostd.persistence.doc.service import ac.uk.ebi.biostd.persistence.common.model.BasicSubmission import ac.uk.ebi.biostd.persistence.common.request.SubmissionFilter import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService +import ac.uk.ebi.biostd.persistence.doc.db.data.FileListDocFileDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionRequestDocDataRepository -import ac.uk.ebi.biostd.persistence.doc.db.repositories.FileListDocFileRepository import ac.uk.ebi.biostd.persistence.doc.db.repositories.getByAccNo import ac.uk.ebi.biostd.persistence.doc.mapping.to.ToExtSubmissionMapper import ac.uk.ebi.biostd.persistence.doc.mapping.to.toExtFile @@ -23,7 +23,7 @@ internal class SubmissionMongoPersistenceQueryService( private val submissionRepo: SubmissionDocDataRepository, private val toExtSubmissionMapper: ToExtSubmissionMapper, private val serializationService: ExtSerializationService, - private val fileListDocFileRepository: FileListDocFileRepository, + private val fileListDocFileRepository: FileListDocFileDocDataRepository, private val requestRepository: SubmissionRequestDocDataRepository, ) : SubmissionPersistenceQueryService { override fun existByAccNo(accNo: String): Boolean { @@ -72,20 +72,24 @@ internal class SubmissionMongoPersistenceQueryService( .plus(findSubmissions(owner, submissionFilter)) } - override fun getReferencedFiles(accNo: String, fileListName: String): List { - val subData = submissionRepo.getSubData(accNo) + override fun getReferencedFiles( + sub: ExtSubmission, + fileListName: String, + ): List { return fileListDocFileRepository - .findAllBySubmissionAccNoAndSubmissionVersionGreaterThanAndFileListName(accNo, 0, fileListName) - .map { it.file.toExtFile(subData.released, subData.relPath) } + .findAllBySubmissionAccNoAndSubmissionVersionGreaterThanAndFileListName(sub.accNo, 0, fileListName) + .map { it.file.toExtFile(sub.released, sub.relPath) } } - override fun findReferencedFile(accNo: String, version: Int, path: String): ExtFile? { - val subData = submissionRepo.getSubData(accNo) + override fun findReferencedFile( + sub: ExtSubmission, + path: String, + ): ExtFile? { return fileListDocFileRepository - .findBySubmissionAccNoAndSubmissionVersionAndFilePath(accNo, version, path) + .findBySubmissionAccNoAndSubmissionVersionAndFilePath(sub.accNo, sub.version, path) .firstOrNull() ?.file - ?.toExtFile(subData.released, submissionRepo.getSubData(accNo).relPath) + ?.toExtFile(sub.released, sub.relPath) } private fun findSubmissions(owner: String, filter: SubmissionFilter): List = diff --git a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/FileListDocFileRepositoryTest.kt b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/FileListDocFileRepositoryTest.kt new file mode 100644 index 0000000000..a8273fb83b --- /dev/null +++ b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/FileListDocFileRepositoryTest.kt @@ -0,0 +1,64 @@ +package ac.uk.ebi.biostd.persistence.doc.db.repositories + +import ac.uk.ebi.biostd.persistence.doc.db.data.FileListDocFileDocDataRepository +import ac.uk.ebi.biostd.persistence.doc.integration.MongoDbReposConfig +import ac.uk.ebi.biostd.persistence.doc.model.FileListDocFile +import ac.uk.ebi.biostd.persistence.doc.model.FireDocFile +import ebi.ac.uk.db.MINIMUM_RUNNING_TIME +import ebi.ac.uk.db.MONGO_VERSION +import ebi.ac.uk.extended.model.ExtFileType +import org.assertj.core.api.Assertions.assertThat +import org.bson.types.ObjectId +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.testcontainers.containers.MongoDBContainer +import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers +import org.testcontainers.utility.DockerImageName +import java.time.Duration + +@ExtendWith(SpringExtension::class) +@Testcontainers +@SpringBootTest(classes = [MongoDbReposConfig::class]) +class FileListDocFileRepositoryTest( + @Autowired private val repository: FileListDocFileDocDataRepository, +) { + + @Test + fun findBySubmissionAccNoAndSubmissionVersionAndFilePath() { + val file = FireDocFile("filename", "filePath", "relPath", "fireId", listOf(), "md5", 1L, ExtFileType.FILE.value) + val fileListFile = FileListDocFile( + id = ObjectId(), + submissionId = ObjectId(), + file = file, + fileListName = "file-list", + index = 0, + submissionVersion = 1, + submissionAccNo = "S-TEST123" + ) + repository.save(fileListFile) + + val result = repository.findBySubmissionAccNoAndSubmissionVersionAndFilePath("S-TEST123", 1, "filePath") + + assertThat(result).containsExactly(fileListFile) + } + + companion object { + @Container + val mongoContainer: MongoDBContainer = MongoDBContainer(DockerImageName.parse(MONGO_VERSION)) + .withStartupCheckStrategy(MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(MINIMUM_RUNNING_TIME))) + + @JvmStatic + @DynamicPropertySource + fun propertySource(register: DynamicPropertyRegistry) { + register.add("spring.data.mongodb.uri") { mongoContainer.getReplicaSetUrl("biostudies-test") } + register.add("spring.data.mongodb.database") { "biostudies-test" } + } + } +} diff --git a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoQueryServiceTest.kt b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoQueryServiceTest.kt index 8c55e99897..b509910f64 100644 --- a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoQueryServiceTest.kt +++ b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoQueryServiceTest.kt @@ -7,10 +7,11 @@ import ac.uk.ebi.biostd.persistence.common.model.RequestStatus.FILES_COPIED import ac.uk.ebi.biostd.persistence.common.model.RequestStatus.LOADED import ac.uk.ebi.biostd.persistence.common.model.RequestStatus.REQUESTED import ac.uk.ebi.biostd.persistence.common.request.SubmissionFilter +import ac.uk.ebi.biostd.persistence.doc.db.data.FileListDocFileDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionDocDataRepository import ac.uk.ebi.biostd.persistence.doc.db.data.SubmissionRequestDocDataRepository -import ac.uk.ebi.biostd.persistence.doc.db.repositories.FileListDocFileRepository import ac.uk.ebi.biostd.persistence.doc.integration.MongoDbReposConfig +import ac.uk.ebi.biostd.persistence.doc.integration.MongoDbServicesConfig import ac.uk.ebi.biostd.persistence.doc.mapping.from.toDocFile import ac.uk.ebi.biostd.persistence.doc.mapping.to.ToExtSubmissionMapper import ac.uk.ebi.biostd.persistence.doc.model.DocAttribute @@ -39,7 +40,6 @@ import ebi.ac.uk.model.constants.ProcessingStatus.PROCESSING import ebi.ac.uk.util.collections.second import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension -import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import org.assertj.core.api.Assertions.assertThat import org.bson.types.ObjectId @@ -51,6 +51,8 @@ import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration import org.springframework.test.context.DynamicPropertyRegistry import org.springframework.test.context.DynamicPropertySource import org.springframework.test.context.junit.jupiter.SpringExtension @@ -61,6 +63,8 @@ import org.testcontainers.junit.jupiter.Testcontainers import org.testcontainers.utility.DockerImageName import uk.ac.ebi.extended.serialization.integration.ExtSerializationConfig.extSerializationService import uk.ac.ebi.extended.serialization.service.ExtSerializationService +import uk.ac.ebi.serialization.common.FilesResolver +import java.nio.file.Files import java.time.Duration.ofSeconds import java.time.Instant import java.time.OffsetDateTime @@ -73,12 +77,12 @@ import ac.uk.ebi.biostd.persistence.doc.test.doc.testDocSubmission as docSubmiss @ExtendWith(MockKExtension::class, SpringExtension::class, TemporaryFolderExtension::class) @Testcontainers -@SpringBootTest(classes = [MongoDbReposConfig::class]) +@SpringBootTest(classes = [MongoDbReposConfig::class, MongoDbServicesConfig::class, TestConfiguration::class]) internal class SubmissionMongoQueryServiceTest( private val tempFolder: TemporaryFolder, - @MockK private val toExtSubmissionMapper: ToExtSubmissionMapper, + @Autowired private val toExtSubmissionMapper: ToExtSubmissionMapper, @Autowired private val submissionRepo: SubmissionDocDataRepository, - @Autowired private val fileListDocFileRepository: FileListDocFileRepository, + @Autowired private val fileListDocFileRepository: FileListDocFileDocDataRepository, @Autowired private val requestRepository: SubmissionRequestDocDataRepository, ) { private val serializationService: ExtSerializationService = extSerializationService() @@ -376,9 +380,9 @@ internal class SubmissionMongoQueryServiceTest( private val nfsReferencedFile = tempFolder.createFile("nfsReferenced.txt") private val nfsFile = createNfsFile("nfsReferenced.txt", "Files/nfsReferenced.txt", nfsReferencedFile) private val nfsFileListFile = FileListDocFile( - ObjectId(), - testDocSubmission.id, - nfsFile.toDocFile(), + id = ObjectId(), + submissionId = testDocSubmission.id, + file = nfsFile.toDocFile(), fileListName = "test-file-list", index = 1, submissionVersion = testDocSubmission.version, @@ -386,15 +390,15 @@ internal class SubmissionMongoQueryServiceTest( ) private val fireReferencedFile = tempFolder.createFile("fireReferenced.txt") private val fireFile = FireFile( - "fire-oid", - null, - false, - "fireReferenced.txt", - "Files/fireReferenced.txt", - fireReferencedFile.md5(), - fireReferencedFile.size(), - FILE, - emptyList() + fireId = "fire-oid", + firePath = null, + published = false, + filePath = "fireReferenced.txt", + relPath = "Files/fireReferenced.txt", + md5 = fireReferencedFile.md5(), + size = fireReferencedFile.size(), + type = FILE, + attributes = emptyList() ) private val fireFileListFile = FileListDocFile( ObjectId(), @@ -418,7 +422,8 @@ internal class SubmissionMongoQueryServiceTest( @Test fun `get referenced files`() { - val files = testInstance.getReferencedFiles(SUB_ACC_NO, "test-file-list") + val submission = testInstance.getExtByAccNo(SUB_ACC_NO) + val files = testInstance.getReferencedFiles(submission, "test-file-list") assertThat(files).hasSize(2) val nfsFile = files.first() as NfsFile @@ -441,7 +446,8 @@ internal class SubmissionMongoQueryServiceTest( submissionRepo.save(innerSectionSubmission) fileListDocFileRepository.save(nfsFileListFile.copy(submissionAccNo = innerSectionSubmission.accNo)) - val files = testInstance.getReferencedFiles("S-REF1", "test-file-list") + val submission = testInstance.getExtByAccNo("S-REF1") + val files = testInstance.getReferencedFiles(submission, "test-file-list") assertThat(files).hasSize(1) assertThat((files.first() as NfsFile).file).isEqualTo(nfsReferencedFile) assertThat(nfsFile.fullPath).isEqualTo(nfsReferencedFile.absolutePath) @@ -449,7 +455,8 @@ internal class SubmissionMongoQueryServiceTest( @Test fun `non existing referenced files`() { - assertThat(testInstance.getReferencedFiles(SUB_ACC_NO, "non-existing-fileListName")).hasSize(0) + val submission = testInstance.getExtByAccNo(SUB_ACC_NO) + assertThat(testInstance.getReferencedFiles(submission, "non-existing-fileListName")).hasSize(0) } } @@ -472,3 +479,11 @@ internal class SubmissionMongoQueryServiceTest( } } } + +@Configuration +class TestConfiguration { + @Bean + fun fileResolver(): FilesResolver { + return FilesResolver(Files.createTempDirectory("sub-requests").toFile()) + } +} diff --git a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionFileSourceTest.kt b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionFileSourceTest.kt index fbedc32cb4..76dc19e71e 100644 --- a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionFileSourceTest.kt +++ b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionFileSourceTest.kt @@ -99,7 +99,7 @@ class SubmissionFileSourceTest( assertThat(webClient.submitSingle(submission("FileList.tsv"), TSV, filesConfig)).isSuccessful() val firstVersion = submissionRepository.getExtByAccNo("S-FSTST1") - val firstVersionReferencedFiles = submissionRepository.getReferencedFiles("S-FSTST1", "FileList") + val firstVersionReferencedFiles = submissionRepository.getReferencedFiles(firstVersion, "FileList") val subFilesPath = "$submissionPath/${firstVersion.relPath}/Files" val innerFile = Paths.get("$subFilesPath/File1.txt") val referencedFile = Paths.get("$subFilesPath/File2.txt") @@ -126,7 +126,7 @@ class SubmissionFileSourceTest( if (enableFire) { val secondVersion = submissionRepository.getExtByAccNo("S-FSTST1") - val secondVersionReferencedFiles = submissionRepository.getReferencedFiles("S-FSTST1", "FileList") + val secondVersionReferencedFiles = submissionRepository.getReferencedFiles(secondVersion, "FileList") val firstVersionFireId = (firstVersion.allSectionsFiles.first() as FireFile).fireId val secondVersionFireId = (secondVersion.allSectionsFiles.first() as FireFile).fireId @@ -179,7 +179,7 @@ class SubmissionFileSourceTest( assertThat(webClient.submitSingle(submission, TSV, filesConfig)).isSuccessful() val persistedSubmission = submissionRepository.getExtByAccNo("S-FSTST2") - val firstVersionReferencedFiles = submissionRepository.getReferencedFiles("S-FSTST2", "FileList") + val firstVersionReferencedFiles = submissionRepository.getReferencedFiles(persistedSubmission, "FileList") val subFilesPath = "$submissionPath/${persistedSubmission.relPath}/Files" val innerFile = Paths.get("$subFilesPath/File4.txt") val referencedFile = Paths.get("$subFilesPath/File3.txt") @@ -273,7 +273,7 @@ class SubmissionFileSourceTest( assertThat(firstVersion.version).isEqualTo(1) assertThat(File("$submissionPath/${firstVersion.relPath}/Files/MultipleReferences.txt")).exists() - val firstVersionReferencedFiles = submissionRepository.getReferencedFiles("S-FSTST4", "FirstVersionFileList") + val firstVersionReferencedFiles = submissionRepository.getReferencedFiles(firstVersion, "FirstVersionFileList") assertThat(firstVersionReferencedFiles).hasSize(2) assertThat(firstVersionReferencedFiles.first().attributes).containsExactly(ExtAttribute("Type", "Ref 1")) assertThat(firstVersionReferencedFiles.second().attributes).containsExactly(ExtAttribute("Type", "Ref 2")) @@ -312,9 +312,9 @@ class SubmissionFileSourceTest( assertThat(secondVersionFiles).hasSize(1) assertThat(secondVersionFiles.first().attributes).containsExactly(ExtAttribute("Type", "Another reference")) - val secondVersionReferencedFiles = submissionRepository.getReferencedFiles("S-FSTST4", "SecondVersionFileList") - assertThat(secondVersionReferencedFiles).hasSize(1) - assertThat(secondVersionReferencedFiles.first().attributes).containsExactly(ExtAttribute("Type", "A reference")) + val secondVersionRefFiles = submissionRepository.getReferencedFiles(secondVersion, "SecondVersionFileList") + assertThat(secondVersionRefFiles).hasSize(1) + assertThat(secondVersionRefFiles.first().attributes).containsExactly(ExtAttribute("Type", "A reference")) } @Test diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt index 2e8b592d30..6064afcc67 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt @@ -89,8 +89,7 @@ class SubmissionConfig( @Bean fun extSubmissionQueryService( queryService: SubmissionPersistenceQueryService, - requestService: SubmissionRequestPersistenceService, - ): ExtSubmissionQueryService = ExtSubmissionQueryService(requestService, queryService) + ): ExtSubmissionQueryService = ExtSubmissionQueryService(queryService) @Bean fun extSubmissionService( diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/ExtSubmissionQueryService.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/ExtSubmissionQueryService.kt index 3084045f45..1df6527b64 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/ExtSubmissionQueryService.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/ExtSubmissionQueryService.kt @@ -2,7 +2,6 @@ package ac.uk.ebi.biostd.submission.domain.service import ac.uk.ebi.biostd.persistence.common.request.SubmissionFilter import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService -import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import ac.uk.ebi.biostd.submission.web.model.ExtPageRequest import ebi.ac.uk.extended.model.ExtFileTable import ebi.ac.uk.extended.model.ExtSubmission @@ -11,10 +10,8 @@ import org.springframework.data.domain.PageImpl import java.time.OffsetDateTime class ExtSubmissionQueryService( - private val requestService: SubmissionRequestPersistenceService, private val submissionPersistenceQueryService: SubmissionPersistenceQueryService, ) { - fun hasActiveRequest(accNo: String): Boolean = requestService.hasActiveRequest(accNo) fun getExtendedSubmission(accNo: String, includeFileListFiles: Boolean = false): ExtSubmission = submissionPersistenceQueryService.getExtByAccNo(accNo, includeFileListFiles) @@ -22,8 +19,11 @@ class ExtSubmissionQueryService( fun findExtendedSubmission(accNo: String, includeFileListFiles: Boolean = false): ExtSubmission? = submissionPersistenceQueryService.findExtByAccNo(accNo, includeFileListFiles) - fun getReferencedFiles(accNo: String, fileListName: String): ExtFileTable = - ExtFileTable(submissionPersistenceQueryService.getReferencedFiles(accNo, fileListName)) + fun getReferencedFiles(accNo: String, fileListName: String): ExtFileTable { + val sub = submissionPersistenceQueryService.getExtByAccNo(accNo, false) + val files = submissionPersistenceQueryService.getReferencedFiles(sub, fileListName) + return ExtFileTable(files) + } fun getExtendedSubmissions(request: ExtPageRequest): Page { val filter = SubmissionFilter( diff --git a/submission/submission-webapp/src/main/resources/application.yml b/submission/submission-webapp/src/main/resources/application.yml index 34cedf2039..1f9d6a00c8 100644 --- a/submission/submission-webapp/src/main/resources/application.yml +++ b/submission/submission-webapp/src/main/resources/application.yml @@ -53,6 +53,8 @@ server: logging: file: name: logs/application_logs.txt + level: + org.springframework.data.mongodb.core.convert.QueryMapper: ERROR # Disable to avoid mapping log population. app: consumers: 10 # number of processing queue consumers. diff --git a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/ext/ExtSubmissionPersistenceQueryServiceTest.kt b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/ext/ExtSubmissionPersistenceQueryServiceTest.kt index d895552b3e..54c8722617 100644 --- a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/ext/ExtSubmissionPersistenceQueryServiceTest.kt +++ b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/ext/ExtSubmissionPersistenceQueryServiceTest.kt @@ -2,7 +2,6 @@ package ac.uk.ebi.biostd.submission.domain.service.ext import ac.uk.ebi.biostd.persistence.common.request.SubmissionFilter import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService -import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import ac.uk.ebi.biostd.submission.domain.service.ExtSubmissionQueryService import ac.uk.ebi.biostd.submission.web.model.ExtPageRequest import ebi.ac.uk.extended.model.ExtCollection @@ -21,10 +20,9 @@ import org.springframework.data.domain.Pageable @ExtendWith(MockKExtension::class) internal class ExtSubmissionPersistenceQueryServiceTest( - @MockK private val requestService: SubmissionRequestPersistenceService, @MockK private val submissionQueryService: SubmissionPersistenceQueryService, ) { - private val testInstance = ExtSubmissionQueryService(requestService, submissionQueryService) + private val testInstance = ExtSubmissionQueryService(submissionQueryService) @Test fun `get ext submission`() { diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactory.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactory.kt index 683f89cd38..dae6232000 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactory.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactory.kt @@ -24,7 +24,6 @@ class FilesSourceFactory( .allInnerSubmissionFiles .groupBy { it.filePath } .mapValues { it.value.first() } - - return SubmissionFilesSource(sub.accNo, sub.version, nfsFiles, fireClient, previousVersionFiles, queryService) + return SubmissionFilesSource(sub, nfsFiles, fireClient, previousVersionFiles, queryService) } } diff --git a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/SubmissionFilesSource.kt b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/SubmissionFilesSource.kt index 9664566b16..6b25060a5b 100644 --- a/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/SubmissionFilesSource.kt +++ b/submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/helpers/SubmissionFilesSource.kt @@ -3,6 +3,7 @@ package ac.uk.ebi.biostd.submission.helpers import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ebi.ac.uk.extended.mapping.from.toExtAttribute import ebi.ac.uk.extended.model.ExtFile +import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.FireFile import ebi.ac.uk.extended.model.NfsFile import ebi.ac.uk.extended.model.copyWithAttributes @@ -14,8 +15,7 @@ import uk.ac.ebi.fire.client.integration.web.FireClient import java.io.File class SubmissionFilesSource( - private val accNo: String, - private val version: Int, + private val sub: ExtSubmission, private val nfsFiles: PathSource, private val fireClient: FireClient, private val previousVersionFiles: Map, @@ -33,7 +33,7 @@ class SubmissionFilesSource( } private fun findSubmissionFile(path: String): ExtFile? { - return previousVersionFiles[path] ?: queryService.findReferencedFile(accNo, version, path) + return previousVersionFiles[path] ?: queryService.findReferencedFile(sub, path) } private fun getFile(file: ExtFile): File? { diff --git a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactoryTest.kt b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactoryTest.kt index 7978888535..223978c15d 100644 --- a/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactoryTest.kt +++ b/submission/submitter/src/test/kotlin/ac/uk/ebi/biostd/submission/helpers/FilesSourceFactoryTest.kt @@ -32,10 +32,11 @@ class FilesSourceFactoryTest( val sub = basicExtSubmission.copy(section = ExtSection(type = "Exp", files = listOf(left(innerFile)))) every { applicationProperties.submissionPath } returns "sub-path" - every { queryService.findReferencedFile(sub.accNo, sub.version, "ghost.txt") } returns null - every { queryService.findReferencedFile(sub.accNo, sub.version, "ref.txt") } returns refFile + every { queryService.findReferencedFile(sub, "ghost.txt") } returns null + every { queryService.findReferencedFile(sub, "ref.txt") } returns refFile val fileSource = testInstance.createSubmissionSource(sub) + assertThat(fileSource.getExtFile("ghost.txt")).isNull() assertThat(fileSource.getExtFile("ref.txt")).isEqualTo(refFile) assertThat(fileSource.getExtFile("inner.txt")).isEqualTo(innerFile)