diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index 1f37f0ab18..0ad58e68dc 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -25,6 +25,7 @@ import Versions.H2Version import Versions.JSONOrgVersion import Versions.JacksonVersion import Versions.JavaValidationApiVersion +import Versions.JetBrainsAnnotationsVersion import Versions.JschVersion import Versions.JwtVersion import Versions.KMongoCoroutineVersion @@ -74,6 +75,7 @@ object Versions { const val SpringVersion = "5.2.20.RELEASE" const val SpringAdminVersion = "2.3.1" const val KotlinVersion = "1.6.10" + const val JetBrainsAnnotationsVersion = "24.0.1" const val KotlinCoroutinesVersion = "1.6.3" const val KotlinLoggingVersion = "1.6.20" @@ -194,6 +196,7 @@ object Dependencies { const val ZipUtil = "org.zeroturnaround:zt-zip:$ZipUtilVersion" // Kotlin specific + const val JetBrainsAnnotations = "org.jetbrains:annotations:$JetBrainsAnnotationsVersion" const val KotlinStdLib = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$KotlinVersion" const val KotlinReflect = "org.jetbrains.kotlin:kotlin-reflect:$KotlinVersion" const val KotlinCoroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-core:$KotlinCoroutinesVersion" diff --git a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtFileListMapper.kt b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtFileListMapper.kt index 0b67398481..a2cddb519f 100644 --- a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtFileListMapper.kt +++ b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtFileListMapper.kt @@ -8,7 +8,6 @@ import ebi.ac.uk.io.sources.FileSourcesList import ebi.ac.uk.io.use import ebi.ac.uk.model.FileList import ebi.ac.uk.model.canonicalName -import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach import mu.KotlinLogging @@ -46,8 +45,7 @@ class ToExtFileListMapper( target: OutputStream, ) { val idx = AtomicInteger(0) - val sourceFiles = serializationService.deserializeFileList(input, format) - .asFlow() + val sourceFiles = serializationService.deserializeFileListAsFlow(input, format) .onEach { file -> logger.info { "$accNo, Mapping file ${idx.getAndIncrement()}, path='${file.path}'" } } .map { sources.getExtFile(it.path, it.type, it.attributes) } val files = extSerializationService.serialize(sourceFiles, target) diff --git a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapper.kt b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapper.kt index 29e8bfae64..9e05f62b5a 100644 --- a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapper.kt +++ b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapper.kt @@ -7,6 +7,7 @@ import ebi.ac.uk.extended.model.ExtFileList import ebi.ac.uk.io.use import ebi.ac.uk.model.FileList import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.map import uk.ac.ebi.extended.serialization.service.ExtSerializationService import uk.ac.ebi.serialization.common.FilesResolver @@ -20,56 +21,41 @@ class ToFileListMapper( private val extSerializationService: ExtSerializationService, private val filesResolver: FilesResolver, ) { - fun convert(fileList: ExtFileList): FileList = FileList(fileList.filePath, emptyFile(fileList.fileName)) + suspend fun convert(fileList: ExtFileList): FileList = FileList(fileList.filePath, emptyFile(fileList.fileName)) - fun serialize(fileList: ExtFileList, targetFormat: SubFormat, file: File): File { + suspend fun serialize(fileList: ExtFileList, targetFormat: SubFormat, file: File): File { toFile(fileList.file, targetFormat, file) return file } - fun serialize(fileListFiles: Sequence, targetFormat: SubFormat, file: File): File { - toFile(fileListFiles, targetFormat, file) - return file - } - suspend fun serialize(fileListFiles: Flow, targetFormat: SubFormat, file: File): File { toFile(fileListFiles, targetFormat, file) return file } - private fun emptyFile(fileName: String): File { + private suspend fun emptyFile(fileName: String): File { val targetFile = filesResolver.createEmptyFile(fileName = fileName) - targetFile.outputStream().use { serializationService.serializeFileList(emptySequence(), SubFormat.JSON, it) } + targetFile.outputStream().use { serializationService.serializeFileList(emptyFlow(), SubFormat.JSON, it) } return targetFile } - private fun toFile(source: File, targetFormat: SubFormat, target: File): File { + private suspend fun toFile(source: File, targetFormat: SubFormat, target: File): File { use(source.inputStream(), target.outputStream()) { input, output -> copy(input, targetFormat, output) } return target } - private fun toFile(source: Sequence, targetFormat: SubFormat, target: File): File { - target.outputStream().use { copy(source, targetFormat, it) } - return target - } - private suspend fun toFile(source: Flow, targetFormat: SubFormat, target: File): File { target.outputStream().use { copy(source, targetFormat, it) } return target } - private fun copy(source: Sequence, targetFormat: SubFormat, target: OutputStream) { - val sourceFiles = source.map { it.toFile() } - serializationService.serializeFileList(sourceFiles, targetFormat, target) - } - private suspend fun copy(source: Flow, targetFormat: SubFormat, target: OutputStream) { val sourceFiles = source.map { it.toFile() } serializationService.serializeFileList(sourceFiles, targetFormat, target) } - private fun copy(input: InputStream, targetFormat: SubFormat, target: OutputStream) { - val sourceFiles = extSerializationService.deserializeList(input).map { it.toFile() } + private suspend fun copy(input: InputStream, targetFormat: SubFormat, target: OutputStream) { + val sourceFiles = extSerializationService.deserializeListAsFlow(input).map { it.toFile() } serializationService.serializeFileList(sourceFiles, targetFormat, target) } } diff --git a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapper.kt b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapper.kt index 31169e2981..dac65d8e12 100644 --- a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapper.kt +++ b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapper.kt @@ -6,7 +6,7 @@ import ebi.ac.uk.model.Section import ebi.ac.uk.model.SectionsTable class ToSectionMapper(private val toFileListMapper: ToFileListMapper) { - fun convert(sec: ExtSection): Section = Section( + suspend fun convert(sec: ExtSection): Section = Section( type = sec.type, accNo = sec.accNo, fileList = sec.fileList?.let { toFileListMapper.convert(it) }, @@ -16,6 +16,6 @@ class ToSectionMapper(private val toFileListMapper: ToFileListMapper) { sections = sec.sections.mapTo(mutableListOf()) { either -> either.bimap({ convert(it) }, { toTable(it) }) } ) - private fun toTable(extSectionTable: ExtSectionTable): SectionsTable = + private suspend fun toTable(extSectionTable: ExtSectionTable): SectionsTable = SectionsTable(extSectionTable.sections.map { section -> convert(section) }) } diff --git a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapper.kt b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapper.kt index 462c99a832..d088b95129 100644 --- a/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapper.kt +++ b/commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapper.kt @@ -16,7 +16,7 @@ class ToSubmissionMapper(private val toSectionMapper: ToSectionMapper) { * Return a simple submission which does not contain submission secret information. Used to generate public * submission tab files. */ - fun toSimpleSubmission(sub: ExtSubmission): Submission = Submission( + suspend fun toSimpleSubmission(sub: ExtSubmission): Submission = Submission( accNo = sub.accNo, section = toSectionMapper.convert(sub.section), attributes = sub.simpleAttributes(), diff --git a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapperTest.kt b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapperTest.kt index bd424b363b..58b27cae74 100644 --- a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapperTest.kt +++ b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapperTest.kt @@ -7,10 +7,15 @@ import ebi.ac.uk.extended.model.ExtFileList import ebi.ac.uk.model.BioFile import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import io.mockk.coEvery import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.mockkStatic +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -31,13 +36,13 @@ internal class ToFileListMapperTest( private val testInstance = ToFileListMapper(serializationService, extSerializationService, filesResolver) @Test - fun convert() { + fun convert() = runTest { every { filesResolver.createEmptyFile("fileList") } returns temporaryFolder.createFile("target-file-list.json") - every { serializationService.serializeFileList(any>(), any(), any()) } answers { - val sequence = arg>(0) + coEvery { serializationService.serializeFileList(any>(), any(), any()) } coAnswers { + val flow = arg>(0) val format = arg(1) val stream = arg(2) - stream.use { it.write("${sequence.toList().size}-$format".toByteArray()) } + stream.use { it.write("${flow.toList().size}-$format".toByteArray()) } } val fileList = testInstance.convert(extFileList) @@ -47,16 +52,16 @@ internal class ToFileListMapperTest( } @Test - fun serialize() { + fun serialize() = runTest { val target = temporaryFolder.createFile("target-file-list.json") mockkStatic(TO_FILE_EXTENSIONS) every { extFile.toFile() } returns bioFile - every { extSerializationService.deserializeList(any()) } returns sequenceOf(extFile) - every { serializationService.serializeFileList(any>(), any(), any()) } answers { - val sequence = arg>(0) + coEvery { extSerializationService.deserializeListAsFlow(any()) } returns flowOf(extFile) + coEvery { serializationService.serializeFileList(any>(), any(), any()) } coAnswers { + val flow = arg>(0) val format = arg(1) val stream = arg(2) - stream.use { it.write("${sequence.toList().size}-$format".toByteArray()) } + stream.use { it.write("${flow.toList().size}-$format".toByteArray()) } } val result = testInstance.serialize(extFileList, SubFormat.XML, target) diff --git a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapperTest.kt b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapperTest.kt index bfbd169db8..e72b00170a 100644 --- a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapperTest.kt +++ b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapperTest.kt @@ -18,11 +18,13 @@ import ebi.ac.uk.model.Link import ebi.ac.uk.model.LinksTable import ebi.ac.uk.model.Section import ebi.ac.uk.util.collections.second +import io.mockk.coEvery import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.mockk import io.mockk.mockkStatic +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -41,7 +43,7 @@ class ToSectionMapperTest( @MockK val extFile: ExtFile, @MockK val extFileTable: ExtFileTable, @MockK val extLink: ExtLink, - @MockK val extLinkTable: ExtLinkTable + @MockK val extLinkTable: ExtLinkTable, ) { private val subExtSection = ExtSection(type = "subtype", accNo = "accNo1") private val subSection = Section(type = "subtype", accNo = "accNo1") @@ -58,7 +60,7 @@ class ToSectionMapperTest( private val testInstance = ToSectionMapper(toFileListMapper) @Test - fun toSection() { + fun toSection() = runTest { mockkStatic( TO_ATTRIBUTE_EXTENSIONS, TO_FILE_EXTENSIONS, @@ -67,7 +69,7 @@ class ToSectionMapperTest( ) { every { extAttribute.toAttribute() } returns attribute every { extFile.toFile() } returns file - every { toFileListMapper.convert(extFileList) } returns fileList + coEvery { toFileListMapper.convert(extFileList) } returns fileList every { extLink.toLink() } returns link every { extFileTable.toTable() } returns fileTable every { extLinkTable.toTable() } returns linkTable diff --git a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapperTest.kt b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapperTest.kt index 84922e21ac..2098530119 100644 --- a/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapperTest.kt +++ b/commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapperTest.kt @@ -11,9 +11,10 @@ import ebi.ac.uk.model.extensions.releaseDate import ebi.ac.uk.model.extensions.rootPath import ebi.ac.uk.model.extensions.title import ebi.ac.uk.test.basicExtSubmission -import io.mockk.every +import io.mockk.coEvery import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -37,9 +38,9 @@ class ToSubmissionMapperTest( private val testInstance = ToSubmissionMapper(toSectionMapper) @Test - fun toSimpleSubmission() { + fun toSimpleSubmission() = runTest { val section = Section() - every { toSectionMapper.convert(extSubmission.section) } returns section + coEvery { toSectionMapper.convert(extSubmission.section) } returns section val submission = testInstance.toSimpleSubmission(extSubmission) diff --git a/commons/commons-model-extended-serialization/build.gradle.kts b/commons/commons-model-extended-serialization/build.gradle.kts index 274ffb26e2..c92b75965e 100644 --- a/commons/commons-model-extended-serialization/build.gradle.kts +++ b/commons/commons-model-extended-serialization/build.gradle.kts @@ -36,4 +36,5 @@ dependencies { BaseTestRuntimeDependencies.forEach { testImplementation(it) } testFixturesApi(Arrow) + testFixturesImplementation(KotlinCoroutines) } diff --git a/commons/commons-model-extended-serialization/gradle.properties b/commons/commons-model-extended-serialization/gradle.properties index 4a1f1c10b0..b3f9e095d6 100644 --- a/commons/commons-model-extended-serialization/gradle.properties +++ b/commons/commons-model-extended-serialization/gradle.properties @@ -1 +1 @@ -coverage=0.81 +coverage=0.78 diff --git a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt index b39ace0655..e2013c2259 100644 --- a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt +++ b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt @@ -39,7 +39,8 @@ import uk.ac.ebi.extended.serialization.serializers.ExtSectionSerializer import uk.ac.ebi.extended.serialization.serializers.ExtSectionsTableSerializer import uk.ac.ebi.extended.serialization.serializers.ExtSubmissionSerializer import uk.ac.ebi.extended.serialization.serializers.OffsetDateTimeSerializer -import uk.ac.ebi.serialization.extensions.deserializeList +import uk.ac.ebi.serialization.extensions.deserializeAsFlow +import uk.ac.ebi.serialization.extensions.deserializeAsSequence import uk.ac.ebi.serialization.extensions.serializeFlow import uk.ac.ebi.serialization.extensions.serializeList import uk.ac.ebi.serialization.serializers.EitherSerializer @@ -54,16 +55,17 @@ data class Properties(val includeFileListFiles: Boolean) : StringWriter() class ExtSerializationService private constructor(val mapper: ObjectMapper) { fun serialize(sub: ExtSubmission, props: Properties = Properties(false)): String = serializeElement(sub, props) fun serialize(files: Sequence, stream: OutputStream): Int = mapper.serializeList(files, stream) - suspend fun serialize(files: Flow, stream: OutputStream): Int = mapper.serializeFlow(files, stream) fun serialize(file: ExtFile): String = serializeElement(file) fun serialize(table: ExtFileTable): String = serializeElement(table) fun serialize(extPage: WebExtPage): String = serializeElement(extPage) + suspend fun serialize(files: Flow, stream: OutputStream): Int = mapper.serializeFlow(files, stream) fun deserialize(value: String): ExtSubmission = mapper.readValue(value) fun deserializeFile(value: String): ExtFile = mapper.readValue(value) - fun deserializeList(stream: InputStream): Sequence = mapper.deserializeList(stream) fun deserializePage(value: String): ExtPage = mapper.readValue(value) fun deserializeTable(value: String): ExtFileTable = mapper.readValue(value) + fun deserializeListAsSequence(stream: InputStream): Sequence = mapper.deserializeAsSequence(stream) + suspend fun deserializeListAsFlow(stream: InputStream): Flow = mapper.deserializeAsFlow(stream) /** * Serialize a generic element. ONLY for testing purpose. diff --git a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt index ce5c2f391a..b912e07230 100644 --- a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt +++ b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt @@ -4,19 +4,21 @@ import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.allFileList import ebi.ac.uk.extended.model.allSectionsFiles +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flow /** - * Return a sequence with all the files of a submission. Pagetab files are retrieved first, followed by section files + * Return a flow with all the files of a submission. Pagetab files are retrieved first, followed by section files * and file list files. */ -fun ExtSerializationService.fileSequence(submission: ExtSubmission): Sequence { - return sequence { - submission.pageTabFiles.forEach { yield(it) } - submission.allFileList.forEach { fileList -> fileList.pageTabFiles.forEach { yield(it) } } - - submission.allSectionsFiles.forEach { yield(it) } +fun ExtSerializationService.filesFlow(submission: ExtSubmission): Flow { + return flow { + submission.pageTabFiles.forEach { emit(it) } + submission.allFileList.forEach { fileList -> fileList.pageTabFiles.forEach { emit(it) } } + submission.allSectionsFiles.forEach { emit(it) } submission.allFileList .map { it.file } - .forEach { it.inputStream().use { stream -> deserializeList(stream).forEach { file -> yield(file) } } } + .forEach { it.inputStream().use { stream -> emitAll(deserializeListAsFlow(stream)) } } } } diff --git a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/FileProcessingService.kt b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/FileProcessingService.kt index dd90a86dfc..dc97945896 100644 --- a/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/FileProcessingService.kt +++ b/commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/FileProcessingService.kt @@ -9,7 +9,6 @@ import ebi.ac.uk.extended.model.ExtSectionTable import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.io.use import ebi.ac.uk.util.collections.mapLeft -import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.map import uk.ac.ebi.serialization.common.FilesResolver import java.io.File @@ -95,7 +94,7 @@ class FileProcessingService( inputFile.inputStream(), outputFile.outputStream() ) { input, output -> - val files = serializationService.deserializeList(input).asFlow() + val files = serializationService.deserializeListAsFlow(input) serializationService.serialize(files.map { processFile(it) }, output) } diff --git a/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExtTest.kt b/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExtTest.kt index 46e2e10f0e..95e7ca3b99 100644 --- a/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExtTest.kt +++ b/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExtTest.kt @@ -15,6 +15,8 @@ import ebi.ac.uk.extended.model.FireFile import ebi.ac.uk.extended.model.StorageMode import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -28,7 +30,7 @@ internal class ExtSerializationServiceExtTest( private val testInstance: ExtSerializationService = ExtSerializationService() @Test - fun fileSequence() { + fun fileFlow() = runTest { val fileList1 = tmpFolder.createFile("f1.json") val files = (4..1000).map { createFireFile(it) } @@ -39,7 +41,7 @@ internal class ExtSerializationServiceExtTest( val submission = createTestSubmission(fileList1, pageTabFile, sectionFile, sectionTableFile) val filesCount = testInstance.serialize(files.asSequence(), fileList1.outputStream()) - val result = testInstance.fileSequence(submission).toList() + val result = testInstance.filesFlow(submission).toList() assertThat(filesCount).isEqualTo(files.size) assertThat(result).containsExactlyInAnyOrder(pageTabFile, sectionFile, sectionTableFile, *files.toTypedArray()) } diff --git a/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceTest.kt b/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceTest.kt index 0ccc793bc1..ff4cb15630 100644 --- a/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceTest.kt +++ b/commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceTest.kt @@ -12,6 +12,8 @@ import ebi.ac.uk.io.ext.md5 import ebi.ac.uk.io.ext.size import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -57,13 +59,14 @@ class ExtSerializationServiceTest(private val tempFolder: TemporaryFolder) { } @Test - fun `serialize - deserialize fileList`() { + fun `serialize - deserialize fileList`() = runTest { val fileList = (1..20_000).map { createNfsFile(it) }.asSequence() val iterator = fileList.iterator() testFile.outputStream().use { testInstance.serialize(fileList, it) } - testFile.inputStream() - .use { testInstance.deserializeList(it).onEach { assertThat(it).isEqualTo(iterator.next()) } } + + val result = testFile.inputStream().use { testInstance.deserializeListAsFlow(it).toList() } + result.onEach { assertThat(it).isEqualTo(iterator.next()) } } private fun createNfsFile(index: Int) = NfsFile( diff --git a/commons/commons-model-extended-serialization/src/testFixtures/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt b/commons/commons-model-extended-serialization/src/testFixtures/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt index 418e5cb112..bbca2a4af2 100644 --- a/commons/commons-model-extended-serialization/src/testFixtures/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt +++ b/commons/commons-model-extended-serialization/src/testFixtures/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt @@ -1,6 +1,8 @@ package uk.ac.ebi.extended.serialization.service import ebi.ac.uk.extended.model.ExtFile +import kotlinx.coroutines.flow.toList import java.io.File -fun ExtSerializationService.files(file: File): List = file.inputStream().use { deserializeList(it).toList() } +suspend fun ExtSerializationService.files(file: File): List = + file.inputStream().use { deserializeListAsFlow(it) }.toList() diff --git a/commons/commons-serialization-util/build.gradle.kts b/commons/commons-serialization-util/build.gradle.kts index 7e0861202f..e2e7adafb2 100644 --- a/commons/commons-serialization-util/build.gradle.kts +++ b/commons/commons-serialization-util/build.gradle.kts @@ -2,6 +2,7 @@ import Dependencies.Arrow import Dependencies.JacksonKotlin import Dependencies.JacksonXml import Dependencies.KotlinCoroutines +import Dependencies.KotlinLogging import Dependencies.KotlinReflect import Dependencies.KotlinStdLib import Projects.CommonsUtil @@ -18,6 +19,7 @@ dependencies { implementation(Woodstox) implementation(KotlinReflect) implementation(KotlinStdLib) + implementation(KotlinLogging) implementation(KotlinCoroutines) BaseTestCompileDependencies.forEach { testImplementation(it) } diff --git a/commons/commons-serialization-util/src/main/kotlin/uk/ac/ebi/serialization/extensions/ObjectMapper.kt b/commons/commons-serialization-util/src/main/kotlin/uk/ac/ebi/serialization/extensions/ObjectMapper.kt index f201a9c4e8..9dbecc2ac1 100644 --- a/commons/commons-serialization-util/src/main/kotlin/uk/ac/ebi/serialization/extensions/ObjectMapper.kt +++ b/commons/commons-serialization-util/src/main/kotlin/uk/ac/ebi/serialization/extensions/ObjectMapper.kt @@ -6,11 +6,18 @@ import com.fasterxml.jackson.databind.JavaType import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.convertValue +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.withContext +import mu.KotlinLogging import java.io.InputStream import java.io.OutputStream import java.util.concurrent.atomic.AtomicInteger +private val logger = KotlinLogging.logger {} + fun ObjectMapper.serializeList(files: Sequence, outputStream: OutputStream): Int { val jsonGenerator = factory.createGenerator(outputStream) jsonGenerator.use { @@ -21,19 +28,24 @@ fun ObjectMapper.serializeList(files: Sequence, outputStream: Outpu } } -suspend fun ObjectMapper.serializeFlow(files: Flow, outputStream: OutputStream): Int { - val jsonGenerator = factory.createGenerator(outputStream) - val count = AtomicInteger(0) - jsonGenerator.use { - it.writeStartArray() - files.collect { file -> - writeValue(it, file) - count.getAndIncrement() +suspend fun ObjectMapper.serializeFlow( + files: Flow, + outputStream: OutputStream, +): Int = + withContext(Dispatchers.IO) { + val jsonGenerator = factory.createGenerator(outputStream) + val count = AtomicInteger(0) + jsonGenerator.use { + it.writeStartArray() + files + .collect { file -> + count.getAndIncrement() + writeValue(it, file) + } + it.writeEndArray() } - it.writeEndArray() - return count.get() + count.get() } -} inline fun ObjectMapper.convertOrDefault(node: JsonNode, property: String, default: () -> T): T = when (val propertyNode: JsonNode? = node.findNode(property)) { @@ -41,7 +53,19 @@ inline fun ObjectMapper.convertOrDefault(node: JsonNode, property: S else -> convertValue(propertyNode) } -inline fun ObjectMapper.deserializeList(inputStream: InputStream): Sequence { +inline fun ObjectMapper.deserializeAsFlow(inputStream: InputStream): Flow { + val jsonParser = factory.createParser(inputStream) + if (jsonParser.nextToken() != JsonToken.START_ARRAY) throw IllegalStateException("Expected content to be an array") + return flow { + var next = jsonParser.nextToken() + while (next != null && next != JsonToken.END_ARRAY) { + emit(readValue(jsonParser, T::class.java)) + next = jsonParser.nextToken() + } + }.flowOn(Dispatchers.IO) +} + +inline fun ObjectMapper.deserializeAsSequence(inputStream: InputStream): Sequence { val jsonParser = factory.createParser(inputStream) if (jsonParser.nextToken() != JsonToken.START_ARRAY) throw IllegalStateException("Expected content to be an array") return asSequence(jsonParser) diff --git a/commons/commons-serialization/build.gradle.kts b/commons/commons-serialization/build.gradle.kts index 1df75cfdd6..4cd0b51fb2 100644 --- a/commons/commons-serialization/build.gradle.kts +++ b/commons/commons-serialization/build.gradle.kts @@ -3,6 +3,7 @@ import Dependencies.CommonsCsv import Dependencies.Guava import Dependencies.JacksonKotlin import Dependencies.JacksonXml +import Dependencies.JetBrainsAnnotations import Dependencies.KotlinCoroutines import Dependencies.KotlinLogging import Dependencies.KotlinReflect @@ -34,6 +35,7 @@ dependencies { api(project(TsvLibrary)) api(project(JsonLibrary)) + compileOnly(JetBrainsAnnotations) implementation(KotlinReflect) implementation(KotlinLogging) implementation(KotlinStdLib) @@ -52,4 +54,6 @@ dependencies { testImplementation(XmlUnitAssertJ) testImplementation(KotlinXmlBuilder) testImplementation(JsonAssert) + + testFixturesImplementation(KotlinCoroutines) } diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/integration/SerializationService.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/integration/SerializationService.kt index e79aff4bbe..75925e3c43 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/integration/SerializationService.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/integration/SerializationService.kt @@ -5,6 +5,7 @@ import ebi.ac.uk.model.BioFile import ebi.ac.uk.model.FilesTable import ebi.ac.uk.model.Submission import kotlinx.coroutines.flow.Flow +import org.jetbrains.annotations.Blocking import java.io.File import java.io.InputStream import java.io.OutputStream @@ -12,19 +13,18 @@ import java.io.OutputStream interface SerializationService { fun serializeSubmission(submission: Submission, format: SubFormat): String - fun serializeTable(table: FilesTable, format: SubFormat, file: File): File + suspend fun serializeTable(table: FilesTable, format: SubFormat, file: File): File - fun serializeFileList(files: Sequence, targetFormat: SubFormat, outputStream: OutputStream) + fun deserializeSubmission(content: String, format: SubFormat): Submission - suspend fun serializeFileList(files: Flow, targetFormat: SubFormat, outputStream: OutputStream) + @Blocking + fun deserializeSubmission(file: File): Submission - fun deserializeSubmission(content: String, format: SubFormat): Submission + fun deserializeFileListAsFlow(inputStream: InputStream, format: SubFormat): Flow - suspend fun deserializeSubmission(content: String, format: SubFormat, source: FileSourcesList): Submission + suspend fun serializeFileList(files: Flow, targetFormat: SubFormat, outputStream: OutputStream) - fun deserializeSubmission(file: File): Submission + suspend fun deserializeSubmission(content: String, format: SubFormat, source: FileSourcesList): Submission suspend fun deserializeSubmission(file: File, source: FileSourcesList): Submission - - fun deserializeFileList(inputStream: InputStream, format: SubFormat): Sequence } diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/json/JsonSerializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/json/JsonSerializer.kt index 761d4a88b8..491beca442 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/json/JsonSerializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/json/JsonSerializer.kt @@ -34,7 +34,7 @@ import ebi.ac.uk.model.Submission import ebi.ac.uk.model.Table import kotlinx.coroutines.flow.Flow import uk.ac.ebi.serialization.deserializers.EitherDeserializer -import uk.ac.ebi.serialization.extensions.deserializeList +import uk.ac.ebi.serialization.extensions.deserializeAsFlow import uk.ac.ebi.serialization.extensions.serializeFlow import uk.ac.ebi.serialization.extensions.serializeList import uk.ac.ebi.serialization.serializers.EitherSerializer @@ -53,7 +53,7 @@ internal class JsonSerializer { suspend fun serializeFileList(fileList: Flow, outputStream: OutputStream) = mapper.serializeFlow(fileList, outputStream) - fun deserializeFileList(inputStream: InputStream): Sequence = mapper.deserializeList(inputStream) + fun deserializeFileList(inputStream: InputStream): Flow = mapper.deserializeAsFlow(inputStream) inline fun deserialize(value: String) = mapper.readValue(value) diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/FileListSerializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/FileListSerializer.kt index 8a51bc764b..277adad203 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/FileListSerializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/FileListSerializer.kt @@ -10,16 +10,18 @@ import ebi.ac.uk.model.FileList import ebi.ac.uk.model.Submission import ebi.ac.uk.model.extensions.allSections import ebi.ac.uk.model.extensions.fileListName +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect import java.io.InputStream internal class FileListSerializer( private val serializer: PagetabSerializer, ) { - internal fun deserializeFileList(inputStream: InputStream, format: SubFormat): Sequence { - return serializer.deserializeFileList(inputStream, format) + internal fun deserializeFileListAsFlow(inputStream: InputStream, format: SubFormat): Flow { + return serializer.deserializeFileListAsFlow(inputStream, format) } - internal suspend fun deserializeFileList(submission: Submission, source: FileSourcesList): Submission { + internal suspend fun deserializeSubmission(submission: Submission, source: FileSourcesList): Submission { submission.allSections() .filter { section -> section.fileListName != null } .map { section -> section to section.fileListName!! } @@ -33,9 +35,9 @@ internal class FileListSerializer( return FileList(name, file) } - private fun checkFileList(name: String, format: SubFormat, stream: InputStream) { + private suspend fun checkFileList(name: String, format: SubFormat, stream: InputStream) { runCatching { - serializer.deserializeFileList(stream, format) + serializer.deserializeFileListAsFlow(stream, format).collect() }.getOrElse { throw InvalidFileListException(name, errorMsg(it)) } diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PageTabSerializationService.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PageTabSerializationService.kt index 0b3a58e546..633cd44b11 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PageTabSerializationService.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PageTabSerializationService.kt @@ -8,6 +8,7 @@ import ebi.ac.uk.model.BioFile import ebi.ac.uk.model.FilesTable import ebi.ac.uk.model.Submission import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow import java.io.File import java.io.InputStream import java.io.OutputStream @@ -28,7 +29,7 @@ internal class PageTabSerializationService( format: SubFormat, source: FileSourcesList, ): Submission = - fileListSerializer.deserializeFileList(serializer.deserializeSubmission(content, format), source) + fileListSerializer.deserializeSubmission(serializer.deserializeSubmission(content, format), source) override fun deserializeSubmission(file: File): Submission { val pagetabFile = readAsPageTab(file) @@ -36,26 +37,18 @@ internal class PageTabSerializationService( } override suspend fun deserializeSubmission(file: File, source: FileSourcesList): Submission = - fileListSerializer.deserializeFileList(deserializeSubmission(file), source) + fileListSerializer.deserializeSubmission(deserializeSubmission(file), source) - override fun deserializeFileList( + override fun deserializeFileListAsFlow( inputStream: InputStream, format: SubFormat, - ): Sequence = fileListSerializer.deserializeFileList(inputStream, format) + ): Flow = fileListSerializer.deserializeFileListAsFlow(inputStream, format) - override fun serializeTable(table: FilesTable, format: SubFormat, file: File): File { - file.outputStream().use { serializer.serializeFileList(table.elements.asSequence(), format, it) } + override suspend fun serializeTable(table: FilesTable, format: SubFormat, file: File): File { + file.outputStream().use { serializer.serializeFileList(table.elements.asFlow(), format, it) } return file } - override fun serializeFileList( - files: Sequence, - targetFormat: SubFormat, - outputStream: OutputStream, - ) { - serializer.serializeFileList(files, targetFormat, outputStream) - } - override suspend fun serializeFileList(files: Flow, targetFormat: SubFormat, outputStream: OutputStream) { serializer.serializeFileList(files, targetFormat, outputStream) } diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PagetabSerializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PagetabSerializer.kt index 905c414e5c..c6345e0426 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PagetabSerializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/service/PagetabSerializer.kt @@ -35,14 +35,6 @@ internal class PagetabSerializer( is TsvFormat -> tsvSerializer.deserializeSubmission(submission) } - fun serializeFileList(files: Sequence, format: SubFormat, outputStream: OutputStream) { - when (format) { - XmlFormat -> xmlStreamSerializer.serializeFileList(files, outputStream) - JsonPretty, PlainJson -> jsonSerializer.serializeFileList(files, outputStream) - is TsvFormat -> tsvSerializer.serializeFileList(files, outputStream) - } - } - suspend fun serializeFileList(files: Flow, format: SubFormat, outputStream: OutputStream) { when (format) { XmlFormat -> xmlStreamSerializer.serializeFileList(files, outputStream) @@ -51,7 +43,7 @@ internal class PagetabSerializer( } } - fun deserializeFileList(input: InputStream, format: SubFormat): Sequence { + fun deserializeFileListAsFlow(input: InputStream, format: SubFormat): Flow { return when (format) { XmlFormat -> xmlStreamSerializer.deserializeFileList(input) is JsonFormat -> jsonSerializer.deserializeFileList(input) diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/TsvSerializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/TsvSerializer.kt index 8e02322a39..f81548f4dc 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/TsvSerializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/TsvSerializer.kt @@ -16,19 +16,9 @@ internal class TsvSerializer( ) { fun serializeSubmission(element: Submission): String = tsvSerializer.serialize(element) - fun deserializeSubmission(pageTab: String): Submission = tsvDeserializer.deserialize(pageTab) - - fun serializeFileList( - files: Sequence, - outputStream: OutputStream, - ) = streamSerializer.serializeFileList(files, outputStream) + suspend fun serializeFileList(files: Flow, outputStream: OutputStream): Unit = + streamSerializer.serializeFileList(files, outputStream) - suspend fun serializeFileList( - files: Flow, - outputStream: OutputStream, - ) = streamSerializer.serializeFileList(files, outputStream) - - fun deserializeFileList( - inputStream: InputStream, - ): Sequence = streamSerializer.deserializeFileList(inputStream) + fun deserializeSubmission(pageTab: String): Submission = tsvDeserializer.deserialize(pageTab) + fun deserializeFileList(inputStream: InputStream): Flow = streamSerializer.deserializeFileList(inputStream) } diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/deserialization/stream/FileListTsvStreamDeserializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/deserialization/stream/FileListTsvStreamDeserializer.kt index 2a8ec7f20c..5aa2db7c2f 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/deserialization/stream/FileListTsvStreamDeserializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/deserialization/stream/FileListTsvStreamDeserializer.kt @@ -8,47 +8,66 @@ import ebi.ac.uk.model.Attribute import ebi.ac.uk.model.BioFile import ebi.ac.uk.model.constants.TableFields.FILES_TABLE import ebi.ac.uk.util.collections.destructure +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.collectIndexed +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.withIndex +import kotlinx.coroutines.withContext +import java.io.BufferedReader import java.io.BufferedWriter import java.io.InputStream import java.io.OutputStream internal class FileListTsvStreamDeserializer { - fun serializeFileList(files: Sequence, fileList: OutputStream) { - val writer = fileList.bufferedWriter() - processFirstFile(files.first(), writer) - files.forEach { file -> writeAttributesValues(file, writer) } - writer.close() + suspend fun serializeFileList(files: Flow, fileList: OutputStream) { + fileList.bufferedWriter().use { it.writeFiles(files) } } - suspend fun serializeFileList(files: Flow, fileList: OutputStream) { - val writer = fileList.bufferedWriter() - processFirstFile(files.first(), writer) - files.collect { file -> writeAttributesValues(file, writer) } - writer.close() + private suspend fun BufferedWriter.writeFiles( + files: Flow, + ) { + files + .collectIndexed { index, file -> + if (index == 0) writeHeaders(file) + writeAttributesValues(file) + } + } + + private suspend fun BufferedWriter.writeHeaders(file: BioFile) = withContext(Dispatchers.IO) { + val attrsNames = file.attributes.map { it.name } + write("Files".plus(TAB).plus(attrsNames.joinToString(TAB.toString()))) + newLine() + } + + private suspend fun BufferedWriter.writeAttributesValues(file: BioFile) = withContext(Dispatchers.IO) { + val attrsValues = file.attributes.map { it.value } + write(file.path.plus(TAB).plus(attrsValues.joinToString(TAB.toString()))) + newLine() } - fun deserializeFileList(fileList: InputStream): Sequence { + fun deserializeFileList(fileList: InputStream): Flow { val reader = fileList.bufferedReader() val (files, headers) = reader.readLine().split(TAB).destructure() if (files != FILES_TABLE.value) throw InvalidElementException("First header value should be 'Files'") - return reader.lineSequence() + return reader + .asFlow() .filter { it.isNotBlank() } - .mapIndexed { index, row -> deserializeRow(index + 1, row.split(TAB), headers) } + .withIndex() + .map { (index, row) -> deserializeRow(index + 1, row.split(TAB), headers) } } - private fun processFirstFile(firstFile: BioFile, writer: BufferedWriter) { - val attrsNames = firstFile.attributes.map { it.name } - writer.write("Files".plus(TAB).plus(attrsNames.joinToString(TAB.toString()))) - writer.newLine() - writeAttributesValues(firstFile, writer) - } - - private fun writeAttributesValues(file: BioFile, writer: BufferedWriter) { - val attrsValues = file.attributes.map { it.value } - writer.write(file.path.plus(TAB).plus(attrsValues.joinToString(TAB.toString()))) - writer.newLine() + private fun BufferedReader.asFlow(): Flow { + return flow { + var line = readLine() + while (line != null) { + emit(line) + line = readLine() + } + }.flowOn(Dispatchers.IO) } private fun deserializeRow(index: Int, row: List, headers: List): BioFile { diff --git a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializer.kt b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializer.kt index a09180cb04..ae5a4c2169 100644 --- a/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializer.kt +++ b/commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializer.kt @@ -2,7 +2,11 @@ package ac.uk.ebi.biostd.xml import com.fasterxml.jackson.dataformat.xml.XmlMapper import ebi.ac.uk.model.BioFile +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.withContext import java.io.InputStream import java.io.OutputStream import javax.xml.stream.XMLInputFactory @@ -17,34 +21,30 @@ import javax.xml.stream.XMLStreamConstants.START_ELEMENT import javax.xml.stream.XMLStreamReader class XmlStreamSerializer { - fun deserializeFileList(inputStream: InputStream): Sequence { + fun deserializeFileList(inputStream: InputStream): Flow { val reader = XMLInputFactory.newFactory().createXMLStreamReader(inputStream) reader.requireEvent(START_DOCUMENT) { "expecting xml document start" } reader.requireEvent(START_ELEMENT, "table") { "expected " } - while (reader.hasNext() && reader.isIgnorable()) reader.next() - return sequence { + return flow { + while (reader.hasNext() && reader.isIgnorable()) { + reader.next() + } + while (reader.eventType == START_ELEMENT && reader.localName == "file") { - yield(XmlSerializer.mapper.readStreamValue(reader)) - while (reader.hasNext() && reader.isIgnorable()) reader.next() + emit(XmlSerializer.mapper.readStreamValue(reader)) + while (reader.hasNext() && reader.isIgnorable()) { + reader.next() + } } reader.requireEvent(END_ELEMENT, "table") { "expected
" } reader.requireEvent(END_DOCUMENT) { "expecting xml document end" } - } - } - - fun serializeFileList(fileList: Sequence, outputStream: OutputStream) { - val streamWriter = XMLOutputFactory.newFactory().createXMLStreamWriter(outputStream) - streamWriter.writeStartDocument() - streamWriter.writeStartElement("table") - fileList.forEach { XmlSerializer.mapper.writeValue(streamWriter, it) } - streamWriter.writeEndElement() - streamWriter.writeEndDocument() + }.flowOn(Dispatchers.IO) } - suspend fun serializeFileList(fileList: Flow, outputStream: OutputStream) { + suspend fun serializeFileList(fileList: Flow, outputStream: OutputStream) = withContext(Dispatchers.IO) { val streamWriter = XMLOutputFactory.newFactory().createXMLStreamWriter(outputStream) streamWriter.writeStartDocument() streamWriter.writeStartElement("table") diff --git a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/JsonSerializerTest.kt b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/JsonSerializerTest.kt index 0af3873e3e..0c06ec36b1 100644 --- a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/JsonSerializerTest.kt +++ b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/JsonSerializerTest.kt @@ -6,13 +6,15 @@ import ebi.ac.uk.model.BioFile import ebi.ac.uk.model.Submission import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @ExtendWith(TemporaryFolderExtension::class) class JsonSerializerTest( - private val temporaryFolder: TemporaryFolder + private val temporaryFolder: TemporaryFolder, ) { private val testInstance = JsonSerializer.mapper private val submission = createVenousBloodMonocyte() @@ -27,19 +29,17 @@ class JsonSerializerTest( } @Test - fun `serialize - deserialize FileList`() { + fun `serialize - deserialize FileList`() = runTest { val jsonSerializer = JsonSerializer() val fileSystem = temporaryFolder.createFile("serialization.json") val files = (1..20_000).map { BioFile("folder$it/file.txt", size = 0L, attributes = attributes(it)) } val iterator = files.iterator() fileSystem.outputStream().use { jsonSerializer.serializeFileList(files.asSequence(), it) } + val response = fileSystem.inputStream().use { jsonSerializer.deserializeFileList(it).toList() } - fileSystem.inputStream().use { - jsonSerializer.deserializeFileList(it).forEach { file -> - assertThat(file).isEqualToComparingFieldByField(iterator.next()) - } - } + assertThat(response).allSatisfy { assertThat(it).isEqualToComparingFieldByField(iterator.next()) } + assertThat(response).hasSize(20_000) } private fun attributes(numberFile: Int) = diff --git a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/deserialization/FileListDeserializerTest.kt b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/deserialization/FileListDeserializerTest.kt index 880068af0f..bfc5eb16ac 100644 --- a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/deserialization/FileListDeserializerTest.kt +++ b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/deserialization/FileListDeserializerTest.kt @@ -8,53 +8,30 @@ import ebi.ac.uk.test.createFile import ebi.ac.uk.util.collections.second import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith -import uk.ac.ebi.serialization.extensions.deserializeList +import uk.ac.ebi.serialization.extensions.deserializeAsFlow @ExtendWith(TemporaryFolderExtension::class) class FileListDeserializerTest( - private val tempFolder: TemporaryFolder + private val tempFolder: TemporaryFolder, ) { private val testInstance = JsonSerializer.mapper @Test - fun `deserialize file list as sequence`() { + fun `deserialize file list as flow`() = runTest { val json = jsonArray( jsonObj { "path" to "File1.txt" }, jsonObj { "path" to "inner/folder" }, ).toString() tempFolder.createFile("FileList.json", json).inputStream().use { - val files = testInstance.deserializeList(it).toList() + val files = testInstance.deserializeAsFlow(it).toList() assertThat(files).hasSize(2) assertThat(files.first().path).isEqualTo("File1.txt") assertThat(files.second().path).isEqualTo("inner/folder") } } - - @Test - fun `sequence iterator idempotent operations`() { - val json = jsonArray( - jsonObj { "path" to "File1.txt" }, - jsonObj { "path" to "File2.txt" }, - ).toString() - - tempFolder.createFile("Iterator.json", json).inputStream().use { - val iterator = testInstance.deserializeList(it).iterator() - assertThat(iterator.hasNext()).isTrue() - assertThat(iterator.hasNext()).isTrue() - - val firstFile = iterator.next() - assertThat(firstFile.path).isEqualTo("File1.txt") - - assertThat(iterator.hasNext()).isTrue() - val secondFile = iterator.next() - assertThat(secondFile.path).isEqualTo("File2.txt") - - assertThat(iterator.hasNext()).isFalse() - assertThrows { iterator.next() } - } - } } diff --git a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/tsv/FileListTsvStreamDeserializerTest.kt b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/tsv/FileListTsvStreamDeserializerTest.kt index 203b8f85be..fef887757e 100644 --- a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/tsv/FileListTsvStreamDeserializerTest.kt +++ b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/tsv/FileListTsvStreamDeserializerTest.kt @@ -1,7 +1,6 @@ package ac.uk.ebi.biostd.tsv import ac.uk.ebi.biostd.tsv.deserialization.stream.FileListTsvStreamDeserializer -import ac.uk.ebi.biostd.validation.InvalidElementException import ac.uk.ebi.biostd.validation.REQUIRED_FILE_PATH import ebi.ac.uk.dsl.tsv.Tsv import ebi.ac.uk.dsl.tsv.line @@ -13,9 +12,11 @@ import ebi.ac.uk.test.createFile import ebi.ac.uk.util.collections.second import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import java.io.File @@ -26,7 +27,7 @@ class FileListTsvStreamDeserializerTest( private val testInstance = FileListTsvStreamDeserializer() @Test - fun `deserialize file list with empty spaces`() { + fun `deserialize file list with empty spaces`() = runTest { val tsvFile = createTsvFile( tsv { line("Files", "Type") @@ -51,7 +52,7 @@ class FileListTsvStreamDeserializerTest( } @Test - fun deserialize() { + fun deserialize() = runTest { val tsvFile = createTsvFile( tsv { line("Files", "Attr1", "Attr2") @@ -80,29 +81,23 @@ class FileListTsvStreamDeserializerTest( } @Test - fun `serialize - deserialize FileList`() { - var idx = 0 - fun bioFile(it: Int): BioFile { - return BioFile( - path = "folder/file$it.txt", - attributes = listOf(Attribute("Attr1", "A$it"), Attribute("Attr2", "B$it")) - ) - } + fun `serialize - deserialize FileList`() = runTest { + fun attributes(numberFile: Int) = + (1..3).map { Attribute(name = "attribute-$it", value = "attribute-$it-file$numberFile-value") } - val files = sequence { yield(bioFile(idx++)) } + val files = (1..20_000).map { BioFile("folder$it/file.txt", size = 0L, attributes = attributes(it)) } + val iterator = files.iterator() val output = tempFolder.createFile("testFile.tsv") - output.outputStream().use { testInstance.serializeFileList(files, it) } + output.outputStream().use { testInstance.serializeFileList(files.asFlow(), it) } - output.inputStream().use { - testInstance.deserializeFileList(it).forEachIndexed { idx, file -> - assertThat(file).isEqualToComparingFieldByField(bioFile(idx)) - } - } + val result = output.inputStream().use { testInstance.deserializeFileList(it).toList() } + assertThat(result).allSatisfy { assertThat(it).isEqualToComparingFieldByField(iterator.next()) } + assertThat(result).hasSize(20000) } @Test - fun `file list with empty path`() { + fun `file list with empty path`() = runTest { val tsv = tsv { line("Files", "Attr1", "Attr2") line("test.txt", "a", "b") @@ -111,9 +106,10 @@ class FileListTsvStreamDeserializerTest( } val testFile = tempFolder.createFile("invalid.tsv", tsv.toString()) - testFile.inputStream().use { - val exception = assertThrows { testInstance.deserializeFileList(it).toList() } - assertThat(exception.message).isEqualTo("Error at row 3: $REQUIRED_FILE_PATH. Element was not created.") + val response = testFile.inputStream().use { runCatching { testInstance.deserializeFileList(it).toList() } } + assertThat(response.isFailure).isTrue() + response.onFailure { + assertThat(it.message).isEqualTo("Error at row 3: $REQUIRED_FILE_PATH. Element was not created.") } } } diff --git a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializerTest.kt b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializerTest.kt index b23fb9550f..fba5e12396 100644 --- a/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializerTest.kt +++ b/commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/xml/XmlStreamSerializerTest.kt @@ -4,29 +4,29 @@ import ebi.ac.uk.model.Attribute import ebi.ac.uk.model.BioFile import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @ExtendWith(TemporaryFolderExtension::class) internal class XmlStreamSerializerTest( - private val temporaryFolder: TemporaryFolder + private val temporaryFolder: TemporaryFolder, ) { val testInstance = XmlStreamSerializer() @Test - fun `serialize - deserialize FileList`() { + fun `serialize - deserialize FileList`() = runTest { val fileSystem = temporaryFolder.createFile("serialization.xml") val files = (1..20_000).map { BioFile("folder$it/file.txt", attributes = attributes(it)) } val iterator = files.iterator() - fileSystem.outputStream().use { testInstance.serializeFileList(files.asSequence(), it) } - - fileSystem.inputStream().use { - testInstance.deserializeFileList(it).forEach { file -> - assertThat(file).isEqualToComparingFieldByField(iterator.next()) - } - } + fileSystem.outputStream().use { testInstance.serializeFileList(files.asFlow(), it) } + val result = fileSystem.inputStream().use { testInstance.deserializeFileList(it).toList() } + assertThat(result).allSatisfy() { assertThat(it).isEqualToComparingFieldByField(iterator.next()) } + assertThat(result).hasSize(20_000) } private fun attributes(numberFile: Int) = diff --git a/commons/commons-serialization/src/testFixtures/kotlin/ac/uk/ebi/biostd/TestFunctions.kt b/commons/commons-serialization/src/testFixtures/kotlin/ac/uk/ebi/biostd/TestFunctions.kt index ac0c3593f2..80906676d2 100644 --- a/commons/commons-serialization/src/testFixtures/kotlin/ac/uk/ebi/biostd/TestFunctions.kt +++ b/commons/commons-serialization/src/testFixtures/kotlin/ac/uk/ebi/biostd/TestFunctions.kt @@ -3,13 +3,15 @@ package ac.uk.ebi.biostd import ac.uk.ebi.biostd.common.SerializationConfig import ac.uk.ebi.biostd.integration.SubFormat import ebi.ac.uk.model.BioFile +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.runBlocking import java.io.File import java.nio.file.Files import kotlin.io.path.outputStream -fun createFileList(vararg files: BioFile, format: SubFormat = SubFormat.JSON): File { +fun createFileList(vararg files: BioFile, format: SubFormat = SubFormat.JSON): File = runBlocking { val file = Files.createTempFile("file-list", "${files.size}-files") val serializer = SerializationConfig.serializationService() - file.outputStream().use { serializer.serializeFileList(files.asSequence(), format, it) } - return file.toFile() + file.outputStream().use { serializer.serializeFileList(files.asFlow(), format, it) } + file.toFile() } diff --git a/commons/commons-transpiler/build.gradle.kts b/commons/commons-transpiler/build.gradle.kts index f50bd140cd..59eda3f2c1 100644 --- a/commons/commons-transpiler/build.gradle.kts +++ b/commons/commons-transpiler/build.gradle.kts @@ -1,4 +1,5 @@ import Dependencies.CliKt +import Dependencies.KotlinCoroutines import Dependencies.KotlinReflect import Dependencies.KotlinStdLib import Projects.CommonsSerialization @@ -21,6 +22,7 @@ dependencies { implementation(CliKt) implementation(KotlinReflect) implementation(KotlinStdLib) + implementation(KotlinCoroutines) testImplementation(Junit) testImplementation(JunitExtensions) diff --git a/commons/commons-transpiler/src/main/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspiler.kt b/commons/commons-transpiler/src/main/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspiler.kt index 20303e020c..b0e298afe7 100644 --- a/commons/commons-transpiler/src/main/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspiler.kt +++ b/commons/commons-transpiler/src/main/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspiler.kt @@ -6,6 +6,7 @@ import ac.uk.ebi.biostd.integration.SubFormat import ac.uk.ebi.transpiler.mapper.FilesTableTemplateMapper import ac.uk.ebi.transpiler.processor.FilesTableTemplateProcessor import ac.uk.ebi.transpiler.validator.FilesTableTemplateValidator +import kotlinx.coroutines.runBlocking import java.nio.file.Files import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.readText @@ -17,7 +18,7 @@ class FilesTableTemplateTranspiler( private val templateProcessor: FilesTableTemplateProcessor = FilesTableTemplateProcessor(), private val templateValidator: FilesTableTemplateValidator = FilesTableTemplateValidator(), private val templateMapper: FilesTableTemplateMapper = FilesTableTemplateMapper(), - private val serializationService: SerializationService = SerializationConfig.serializationService() + private val serializationService: SerializationService = SerializationConfig.serializationService(), ) { /** * Transforms a files table template to its corresponding files table page tab representation in the desired format. @@ -37,8 +38,8 @@ class FilesTableTemplateTranspiler( baseColumns: List, filesPath: String, basePath: String, - format: SubFormat - ): String { + format: SubFormat, + ): String = runBlocking { val tableTemplate = templateProcessor.process(template, baseColumns) templateValidator.validate(tableTemplate, filesPath) @@ -46,6 +47,6 @@ class FilesTableTemplateTranspiler( val filesTable = templateMapper.map(tableTemplate, filesPath, basePath) val file = Files.createTempFile("tempFile.txt", "") serializationService.serializeTable(filesTable, format, file.toFile()) - return file.readText() + file.readText() } } diff --git a/commons/commons-transpiler/src/test/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspilerTest.kt b/commons/commons-transpiler/src/test/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspilerTest.kt index c1bd3049a4..5afa267230 100644 --- a/commons/commons-transpiler/src/test/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspilerTest.kt +++ b/commons/commons-transpiler/src/test/kotlin/ac/uk/ebi/transpiler/service/FilesTableTemplateTranspilerTest.kt @@ -11,11 +11,12 @@ import ac.uk.ebi.transpiler.validator.FilesTableTemplateValidator import ebi.ac.uk.model.FilesTable import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.slot -import io.mockk.verify import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -28,7 +29,7 @@ class FilesTableTemplateTranspilerTest( @MockK private val templateValidator: FilesTableTemplateValidator, @MockK private val templateMapper: FilesTableTemplateMapper, @MockK private val serializationService: SerializationService, - temporaryFolder: TemporaryFolder + temporaryFolder: TemporaryFolder, ) { private val testFile = temporaryFolder.createFile("fileSerialization.txt") private val slotTempFile = slot() @@ -47,7 +48,7 @@ class FilesTableTemplateTranspilerTest( every { templateProcessor.process(testTemplate, testBaseColumns) } returns testFilesTableTemplate every { templateValidator.validate(testFilesTableTemplate, testFilesPath) }.answers { nothing } every { templateMapper.map(testFilesTableTemplate, testFilesPath, testParentFolder) } returns testFilesTable - every { serializationService.serializeTable(testFilesTable, TSV, capture(slotTempFile)) } returns testFile + coEvery { serializationService.serializeTable(testFilesTable, TSV, capture(slotTempFile)) } returns testFile } @Test @@ -55,7 +56,7 @@ class FilesTableTemplateTranspilerTest( val result = testInstance.transpile(testTemplate, testBaseColumns, testFilesPath, testParentFolder, Tsv) assertThat(result).isEqualTo("") - verify(exactly = 1) { + coVerify(exactly = 1) { templateProcessor.process(testTemplate, testBaseColumns) templateValidator.validate(testFilesTableTemplate, testFilesPath) templateMapper.map(testFilesTableTemplate, testFilesPath, testParentFolder) diff --git a/scheduler/tasks/exporter-task/src/main/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterService.kt b/scheduler/tasks/exporter-task/src/main/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterService.kt index e41a6564fe..13b8a71495 100644 --- a/scheduler/tasks/exporter-task/src/main/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterService.kt +++ b/scheduler/tasks/exporter-task/src/main/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterService.kt @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import ebi.ac.uk.extended.mapping.to.ToSubmissionMapper import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.isCollection +import kotlinx.coroutines.runBlocking import mu.KotlinLogging import uk.ac.ebi.scheduler.pmc.exporter.config.ApplicationProperties import java.nio.file.Files @@ -45,15 +46,14 @@ class PublicOnlyExporterService( private fun writeSubmissions() { bioWebClient .getExtSubmissionsAsSequence(ExtPageQuery(released = true, limit = 1)) + .filter { it.isCollection.not() } .forEach(::writeSubmission) } - private fun writeSubmission(extSubmission: ExtSubmission) { - if (extSubmission.isCollection.not()) { - logger.info { "Exporting public submission '${extSubmission.accNo}'" } - val simpleSubmission = toSubmissionMapper.toSimpleSubmission(extSubmission) - jsonWriter.writeRawValue(serializationService.serializeSubmission(simpleSubmission, JSON_PRETTY)) - } + private fun writeSubmission(extSubmission: ExtSubmission) = runBlocking { + logger.info { "Exporting public submission '${extSubmission.accNo}'" } + val simpleSubmission = toSubmissionMapper.toSimpleSubmission(extSubmission) + jsonWriter.writeRawValue(serializationService.serializeSubmission(simpleSubmission, JSON_PRETTY)) } private fun closeExportFile() { diff --git a/scheduler/tasks/exporter-task/src/test/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterServiceTest.kt b/scheduler/tasks/exporter-task/src/test/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterServiceTest.kt index 3bc759e2b1..94c46d3c66 100644 --- a/scheduler/tasks/exporter-task/src/test/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterServiceTest.kt +++ b/scheduler/tasks/exporter-task/src/test/kotlin/uk/ac/ebi/scheduler/pmc/exporter/service/PublicOnlyExporterServiceTest.kt @@ -12,6 +12,7 @@ import ebi.ac.uk.test.basicExtSubmission import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension import io.mockk.clearAllMocks +import io.mockk.coEvery import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension @@ -72,7 +73,7 @@ class PublicOnlyExporterServiceTest( private fun mockSerializationService() { val serializedSubmission = jsonObj { "accNo" to "S-TEST123" } - every { + coEvery { serializationService.serializeSubmission( toSubmissionMapper.toSimpleSubmission(basicExtSubmission), JSON_PRETTY diff --git a/scheduler/tasks/pmc-processor-task/gradle.properties b/scheduler/tasks/pmc-processor-task/gradle.properties index c8c897c047..dafbd7a91f 100644 --- a/scheduler/tasks/pmc-processor-task/gradle.properties +++ b/scheduler/tasks/pmc-processor-task/gradle.properties @@ -1 +1 @@ -coverage=0.92 +coverage=0.91 diff --git a/submission/persistence-filesystem/build.gradle.kts b/submission/persistence-filesystem/build.gradle.kts index 53e4113125..91ddf59193 100644 --- a/submission/persistence-filesystem/build.gradle.kts +++ b/submission/persistence-filesystem/build.gradle.kts @@ -1,4 +1,5 @@ import Dependencies.Arrow +import Dependencies.KotlinCoroutines import Dependencies.KotlinLogging import Dependencies.KotlinReflect import Dependencies.KotlinStdLib @@ -42,6 +43,7 @@ dependencies { implementation(KotlinStdLib) implementation(KotlinReflect) implementation(KotlinLogging) + implementation(KotlinCoroutines) implementation(ZipUtil) testImplementation(project(CommonsTest)) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt index c7e50e3555..834929bc27 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FileStorageService.kt @@ -3,6 +3,7 @@ package ac.uk.ebi.biostd.persistence.filesystem.api import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.StorageMode +import kotlinx.coroutines.flow.Flow interface FileStorageService { suspend fun releaseSubmissionFile(file: ExtFile, subRelPath: String, mode: StorageMode): ExtFile @@ -13,6 +14,6 @@ interface FileStorageService { suspend fun deleteSubmissionFiles( sub: ExtSubmission, - process: (Sequence) -> Sequence = { it }, + process: (Flow) -> Flow = { it }, ) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt index 92f0391ad9..e20ea632eb 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/FilesService.kt @@ -8,7 +8,7 @@ internal interface FilesService { suspend fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) - fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) + suspend fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) - fun deleteEmptyFolders(current: ExtSubmission) + suspend fun deleteEmptyFolders(current: ExtSubmission) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/PageTabService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/PageTabService.kt index 7662fae1f2..ddca3d814e 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/PageTabService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/api/PageTabService.kt @@ -3,5 +3,5 @@ package ac.uk.ebi.biostd.persistence.filesystem.api import ebi.ac.uk.extended.model.ExtSubmission internal interface PageTabService { - fun generatePageTab(sub: ExtSubmission): ExtSubmission + suspend fun generatePageTab(sub: ExtSubmission): ExtSubmission } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt index 4649df5d5c..4029a80941 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/fire/FireFilesService.kt @@ -67,11 +67,11 @@ class FireFilesService( client.delete(file.fireId) } - override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { + override suspend fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { // No need to delete FTP links on FIRE as file deleting complete this } - override fun deleteEmptyFolders(current: ExtSubmission) { + override suspend fun deleteEmptyFolders(current: ExtSubmission) { // No need to delete FIRE empty bucket as they only exists as files are in them } } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt index 2db645d223..607c32378f 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesService.kt @@ -14,6 +14,8 @@ import ebi.ac.uk.io.Permissions import ebi.ac.uk.io.RWXR_XR_X import ebi.ac.uk.io.ext.notExist import ebi.ac.uk.paths.SubmissionFolderResolver +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import mu.KotlinLogging import uk.ac.ebi.fire.client.integration.web.FireClient import java.io.File @@ -25,12 +27,13 @@ class NfsFilesService( private val fireClient: FireClient, private val folderResolver: SubmissionFolderResolver, ) : FilesService { - override suspend fun persistSubmissionFile(sub: ExtSubmission, file: ExtFile): ExtFile { - return when (file) { - is FireFile -> persistFireFile(sub, file) - is NfsFile -> persistNfsFile(sub, file) + override suspend fun persistSubmissionFile(sub: ExtSubmission, file: ExtFile): ExtFile = + withContext(Dispatchers.IO) { + when (file) { + is FireFile -> persistFireFile(sub, file) + is NfsFile -> persistNfsFile(sub, file) + } } - } private fun persistNfsFile(sub: ExtSubmission, file: NfsFile): ExtFile { val permissions = sub.permissions() @@ -62,7 +65,7 @@ class NfsFilesService( return getOrCreateFolder(submissionPath, permissions).toFile() } - override suspend fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) { + override suspend fun deleteSubmissionFile(sub: ExtSubmission, file: ExtFile) = withContext(Dispatchers.IO) { require(file is NfsFile) { "NfsFilesService should only handle NfsFile" } val subDirectory = folderResolver.getSubFolder(sub.relPath) @@ -70,14 +73,14 @@ class NfsFilesService( FileUtils.deleteFile(subFile) } - override fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) { + override suspend fun deleteFtpFile(sub: ExtSubmission, file: ExtFile) = withContext(Dispatchers.IO) { logger.info { "${sub.accNo} ${sub.owner} Started un-publishing files of submission ${sub.accNo} on NFS" } val subFolder = folderResolver.getSubmissionFtpFolder(sub.relPath) FileUtils.deleteFile(subFolder.resolve(file.relPath).toFile()) logger.info { "${sub.accNo} ${sub.owner} Finished un-publishing files of submission ${sub.accNo} on NFS" } } - override fun deleteEmptyFolders(current: ExtSubmission) { + override suspend fun deleteEmptyFolders(current: ExtSubmission) = withContext(Dispatchers.IO) { val subFolder = folderResolver.getSubFolder(current.relPath) FileUtils.deleteEmptyDirectories(subFolder.toFile()) } diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabService.kt index 54076ae4f3..9669900af1 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabService.kt @@ -16,7 +16,7 @@ class PageTabService( private val baseTempFolder: File, private val pageTabUtil: PageTabUtil, ) : PageTabService { - override fun generatePageTab(sub: ExtSubmission): ExtSubmission { + override suspend fun generatePageTab(sub: ExtSubmission): ExtSubmission { val tempFolder = createTempFolder(sub.accNo, sub.version) val subFiles = pageTabUtil.generateSubPageTab(sub, tempFolder) val fileListFiles = pageTabUtil.generateFileListPageTab(sub, tempFolder) diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabUtil.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabUtil.kt index c2177d07c1..26d173b4a3 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabUtil.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabUtil.kt @@ -20,7 +20,7 @@ class PageTabUtil( private val toSubmissionMapper: ToSubmissionMapper, private val fileListMapper: ToFileListMapper, ) { - fun generateSubPageTab(sub: ExtSubmission, target: File): PageTabFiles { + suspend fun generateSubPageTab(sub: ExtSubmission, target: File): PageTabFiles { val element = toSubmissionMapper.toSimpleSubmission(sub) val permissions = sub.permissions() @@ -43,10 +43,10 @@ class PageTabUtil( ) } - fun generateFileListPageTab(submission: ExtSubmission, filesFolder: File): Map = + suspend fun generateFileListPageTab(submission: ExtSubmission, filesFolder: File): Map = submission.allFileList.associate { it.filePath to saveTabFiles(filesFolder, it) } - private fun saveTabFiles(filesDir: File, fileList: ExtFileList): PageTabFiles { + private suspend fun saveTabFiles(filesDir: File, fileList: ExtFileList): PageTabFiles { createFolderStructure(filesDir, fileList.filePath) val path = fileList.filePath return PageTabFiles( diff --git a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt index 7cc17463f0..1c315d1442 100644 --- a/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt +++ b/submission/persistence-filesystem/src/main/kotlin/ac/uk/ebi/biostd/persistence/filesystem/service/StorageService.kt @@ -10,9 +10,10 @@ import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.StorageMode import ebi.ac.uk.extended.model.StorageMode.FIRE import ebi.ac.uk.extended.model.StorageMode.NFS +import kotlinx.coroutines.flow.Flow import mu.KotlinLogging import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow private val logger = KotlinLogging.logger {} @@ -51,13 +52,13 @@ class StorageService( override suspend fun deleteSubmissionFiles( sub: ExtSubmission, - process: (Sequence) -> Sequence, + process: (Flow) -> Flow, ) { - process(serializationService.fileSequence(sub)).forEach { file -> deleteSubmissionFile(sub, file) } + process(serializationService.filesFlow(sub)).collect { file -> deleteSubmissionFile(sub, file) } deleteEmptyFolders(sub) } - private fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) { + private suspend fun deleteEmptyFolders(sub: ExtSubmission) = when (sub.storageMode) { FIRE -> fireFilesService.deleteEmptyFolders(sub) NFS -> nfsFilesService.deleteEmptyFolders(sub) } diff --git a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt index 414acc066e..fbfb1ad780 100644 --- a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt +++ b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/nfs/NfsFilesServiceTest.kt @@ -63,7 +63,7 @@ class NfsFilesServiceTest( } @Test - fun `delete ftp links`() { + fun `delete ftp links`() = runTest { val ftpFolder = ftpFolder.createDirectory("S-BSST2") val filesFtpFolder = ftpFolder.createDirectory("Files") val ftpFile = filesFtpFolder.createFile("file1.txt") diff --git a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabServiceTest.kt b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabServiceTest.kt index 6c7598a21b..0e17234cf2 100644 --- a/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabServiceTest.kt +++ b/submission/persistence-filesystem/src/test/kotlin/ac/uk/ebi/biostd/persistence/filesystem/pagetab/PageTabServiceTest.kt @@ -13,10 +13,12 @@ import ebi.ac.uk.util.collections.second import ebi.ac.uk.util.collections.third import io.github.glytching.junit.extension.folder.TemporaryFolder import io.github.glytching.junit.extension.folder.TemporaryFolderExtension +import io.mockk.coEvery import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.mockkStatic +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -40,7 +42,7 @@ internal class PageTabServiceTest( private val testInstance = PageTabService(baseTempDir, pageTabUtil) @Test - fun `generate pagetab`() { + fun `generate pagetab`() = runTest { mockkStatic(LocalDate::class) every { LocalDate.now() } returns LocalDate.of(2023, 1, 24) @@ -50,8 +52,8 @@ internal class PageTabServiceTest( val rootSection = ExtSection(type = "t1", sections = listOf(Either.left(fileListSection))) val sub = basicExtSubmission.copy(section = rootSection) - every { pageTabUtil.generateSubPageTab(sub, tempDir) } returns PageTabFiles(subJson, subXml, subTsv) - every { pageTabUtil.generateFileListPageTab(sub, tempDir) } returns mapOf( + coEvery { pageTabUtil.generateSubPageTab(sub, tempDir) } returns PageTabFiles(subJson, subXml, subTsv) + coEvery { pageTabUtil.generateFileListPageTab(sub, tempDir) } returns mapOf( "a-path" to PageTabFiles( fileListJson, fileListXml, diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt index 09f6ea8576..9a08950685 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/FileListDocFileDocDataRepository.kt @@ -1,13 +1,8 @@ package ac.uk.ebi.biostd.persistence.doc.db.data -import ac.uk.ebi.biostd.persistence.doc.db.converters.shared.FileListDocFileFields.FILE_LIST_DOC_FILE_INDEX import ac.uk.ebi.biostd.persistence.doc.db.reactive.repositories.FileListDocFileRepository import ac.uk.ebi.biostd.persistence.doc.model.FileListDocFile import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.flow.toList -import org.springframework.data.domain.PageRequest -import org.springframework.data.domain.Sort class FileListDocFileDocDataRepository( private val fileListDocFileRepository: FileListDocFileRepository, @@ -18,44 +13,24 @@ class FileListDocFileDocDataRepository( version: Int, fileListName: String, ): Flow { - fun getPaged(page: Int, size: Int): Flow { - return fileListDocFileRepository - .findAllBySubmissionAccNoAndSubmissionVersionGreaterThanAndFileListName( - accNo, - version, - fileListName, - PageRequest.of(page, size, Sort.by(Sort.Direction.ASC, FILE_LIST_DOC_FILE_INDEX)) - ) - } - return pageResultAsFlow(function = { page, size -> getPaged(page, size) }) + return fileListDocFileRepository + .findAllBySubmissionAccNoAndSubmissionVersionGreaterThanAndFileListNameOrderByIndexAsc( + accNo, + version, + fileListName, + ) } - fun findAllBySubmissionAccNoAndSubmissionVersionAndFileListName( + fun findByFileList( accNo: String, version: Int, fileListName: String, ): Flow { - fun getPaged(page: Int, size: Int): Flow { - return fileListDocFileRepository - .findAllBySubmissionAccNoAndSubmissionVersionAndFileListName( - accNo, - version, - fileListName, - PageRequest.of(page, size, Sort.by(Sort.Direction.ASC, FILE_LIST_DOC_FILE_INDEX)) - ) - } - return pageResultAsFlow(function = { page, size -> getPaged(page, size) }) - } -} - -private fun pageResultAsFlow(page: Int = 0, limit: Int = 10, function: (Int, Int) -> Flow): Flow { - - return flow { - var cPage = page - var result = function(cPage, limit).toList() - while (result.isNotEmpty()) { - result.forEach { emit(it) } - result = function(++cPage, limit).toList() - } + return fileListDocFileRepository + .findAllBySubmissionAccNoAndSubmissionVersionAndFileListNameOrderByIndexAsc( + accNo, + version, + fileListName, + ) } } diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/Repositories.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/Repositories.kt index 02875c74a9..4d3aeb4afb 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/Repositories.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/Repositories.kt @@ -111,6 +111,12 @@ interface FileListDocFileRepository : CoroutineCrudRepository + fun findAllBySubmissionAccNoAndSubmissionVersionGreaterThanAndFileListNameOrderByIndexAsc( + accNo: String, + version: Int, + fileListName: String, + ): Flow + fun findAllBySubmissionAccNoAndSubmissionVersionAndFileListName( accNo: String, version: Int, @@ -118,6 +124,12 @@ interface FileListDocFileRepository : CoroutineCrudRepository + fun findAllBySubmissionAccNoAndSubmissionVersionAndFileListNameOrderByIndexAsc( + accNo: String, + version: Int, + fileListName: String, + ): Flow + @Query("{ 'submissionAccNo': ?0, 'submissionVersion': ?1, 'file.filePath': ?2}") suspend fun findBySubmissionAccNoAndSubmissionVersionAndFilePath( accNo: String, diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/from/ToDocFileMapper.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/from/ToDocFileMapper.kt index 3c2407e957..34b9d5a72c 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/from/ToDocFileMapper.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/from/ToDocFileMapper.kt @@ -19,13 +19,13 @@ import java.io.InputStream internal fun Either.toDocFiles() = bimap(ExtFile::toDocFile, ExtFileTable::toDocFileTable) class ToDocFileListMapper( - private val serializationService: ExtSerializationService + private val serializationService: ExtSerializationService, ) { internal fun convert( extFileList: ExtFileList, subId: ObjectId, accNo: String, - version: Int + version: Int, ): Pair> { val listFiles = extFileList.file.inputStream().use { getFiles(it, extFileList.filePath, subId, accNo, version) } val pageTabFiles = extFileList.pageTabFiles.map { it.toDocFile() } @@ -37,9 +37,9 @@ class ToDocFileListMapper( path: String, subId: ObjectId, accNo: String, - version: Int + version: Int, ): List = - serializationService.deserializeList(stream) + serializationService.deserializeListAsSequence(stream) .mapIndexed { idx, file -> FileListDocFile(ObjectId(), subId, file.toDocFile(), path, idx, version, accNo) } .toList() } @@ -57,6 +57,7 @@ internal fun ExtFile.toDocFile(): DocFile = when (this) { fileSize = size, fileType = type.value ) + is NfsFile -> NfsDocFile( fileName = fileName, filePath = filePath, diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapper.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapper.kt index ba08b75436..e3abb699fa 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapper.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapper.kt @@ -4,13 +4,18 @@ import ac.uk.ebi.biostd.persistence.doc.db.data.FileListDocFileDocDataRepository import ac.uk.ebi.biostd.persistence.doc.model.DocFileList import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtFileList +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.map +import mu.KotlinLogging import uk.ac.ebi.extended.serialization.service.ExtSerializationService import uk.ac.ebi.serialization.common.FilesResolver import java.io.File +private val logger = KotlinLogging.logger {} + class ToExtFileListMapper( private val fileListDocFileDocDataRepository: FileListDocFileDocDataRepository, private val serializationService: ExtSerializationService, @@ -31,8 +36,9 @@ class ToExtFileListMapper( ): ExtFileList { fun fileListFiles(): Flow { return fileListDocFileDocDataRepository - .findAllBySubmissionAccNoAndSubmissionVersionAndFileListName(subAccNo, subVersion, fileList.fileName) + .findByFileList(subAccNo, subVersion, fileList.fileName) .map { it.file.toExtFile(released, subRelPath) } + .flowOn(Dispatchers.Default) } val files = if (includeFileListFiles) fileListFiles() else emptyFlow() @@ -43,9 +49,11 @@ class ToExtFileListMapper( ) } - private suspend fun writeFile(subAccNo: String, subVersion: Int, fileListName: String, files: Flow): File { - val file = extFilesResolver.createExtEmptyFile(subAccNo, subVersion, fileListName) + private suspend fun writeFile(accNo: String, version: Int, fileListName: String, files: Flow): File { + logger.info { "accNo:'$accNo' version: '$version', serializing file list '$fileListName'" } + val file = extFilesResolver.createExtEmptyFile(accNo, version, fileListName) file.outputStream().use { serializationService.serialize(files, it) } + logger.info { "accNo:'$accNo' version: '$version', completed file list '$fileListName' serialization" } return file } } diff --git a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/SubmissionDocDataRepositoryTest.kt b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/SubmissionDocDataRepositoryTest.kt index f31896fcb8..8ba9660ce3 100644 --- a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/SubmissionDocDataRepositoryTest.kt +++ b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/db/data/SubmissionDocDataRepositoryTest.kt @@ -101,12 +101,12 @@ internal class SubmissionDocDataRepositoryTest( assertThat(r1).isEmpty() val r2 = fileListDocFileRepo - .findAllBySubmissionAccNoAndSubmissionVersionAndFileListName("S-BSST4", -1, "file-list") + .findByFileList("S-BSST4", -1, "file-list") .toList() assertThat(r2).hasSize(1) val r3 = fileListDocFileRepo - .findAllBySubmissionAccNoAndSubmissionVersionAndFileListName("S-BSST4", -2, "file-list") + .findByFileList("S-BSST4", -2, "file-list") .toList() assertThat(r3).hasSize(1) } diff --git a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapperTest.kt b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapperTest.kt index 6b35209e69..39f1e90100 100644 --- a/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapperTest.kt +++ b/submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/mapping/to/ToExtFileListMapperTest.kt @@ -63,7 +63,7 @@ class ToExtFileListMapperTest(temporaryFolder: TemporaryFolder) { @Test fun `toExtFileList including FileListFiles`() = runTest { every { - fileListDocFileRepository.findAllBySubmissionAccNoAndSubmissionVersionAndFileListName( + fileListDocFileRepository.findByFileList( "S-TEST123", 1, "file-list" diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/stats/SubmissionStatsService.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/stats/SubmissionStatsService.kt index 62ac7ddcc3..236d99823f 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/stats/SubmissionStatsService.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/stats/SubmissionStatsService.kt @@ -8,11 +8,12 @@ import ac.uk.ebi.biostd.persistence.common.service.StatsDataService import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService import ac.uk.ebi.biostd.persistence.doc.model.SingleSubmissionStat import ac.uk.ebi.biostd.submission.helpers.TempFileGenerator +import ebi.ac.uk.extended.model.ExtSubmission import kotlinx.coroutines.flow.Flow import mu.KotlinLogging import org.springframework.web.multipart.MultipartFile import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow private val logger = KotlinLogging.logger {} @@ -60,10 +61,15 @@ class SubmissionStatsService( val sub = extSubmissionQueryService.getExtByAccNo(accNo, includeFileListFiles = true) logger.info { "${sub.accNo} ${sub.owner} Started calculating submission stats" } - val subFilesSize = serializationService.fileSequence(sub).sumOf { it.size } + var subFilesSize = totalSize(sub) val subFilesSizeStat = submissionStatsService.save(SingleSubmissionStat(sub.accNo, subFilesSize, FILES_SIZE)) - logger.info { "${sub.accNo} ${sub.owner} Finished calculating submission stats. Files size: $subFilesSize" } return subFilesSizeStat } + + private suspend fun totalSize(submission: ExtSubmission): Long { + var subFilesSize = 0L + serializationService.filesFlow(submission).collect { subFilesSize = subFilesSize + it.size } + return subFilesSize + } } diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt index c807fcd1f1..2f2e50663e 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleaner.kt @@ -9,11 +9,12 @@ import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.StorageMode import ebi.ac.uk.extended.model.storageMode +import kotlinx.coroutines.flow.collectIndexed +import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.toList import mu.KotlinLogging import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow private val logger = KotlinLogging.logger {} @@ -52,18 +53,19 @@ class SubmissionRequestCleaner( val newFiles = newFilesMap(new) logger.info { "${current.accNo} ${current.owner} Started cleaning common submission files" } - serializationService.fileSequence(current) + serializationService.filesFlow(current) .filter { shouldDelete(newFiles, it) } - .forEachIndexed { index, file -> deleteFile(index, file) } + .collectIndexed { index, file -> deleteFile(index, file) } logger.info { "${current.accNo} ${current.owner} Finished cleaning common submission files" } } private suspend fun newFilesMap(new: ExtSubmission): Map { - return filesRequestService + val response = mutableMapOf() + filesRequestService .getSubmissionRequestFiles(new.accNo, new.version, 0) .map { it.file } - .toList() - .associate { it.filePath to FileEntry(it.md5, new.storageMode) } + .collect { response[it.filePath] = FileEntry(it.md5, new.storageMode) } + return response } private data class FileEntry(val md5: String, val storageMode: StorageMode) diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt index c1ed97e988..e9e0dbc00e 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestFinalizer.kt @@ -7,10 +7,16 @@ import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.storageMode +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.toSet +import kotlinx.coroutines.flow.withIndex import mu.KotlinLogging import uk.ac.ebi.events.service.EventsPublisherService import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow private val logger = KotlinLogging.logger {} @@ -39,10 +45,12 @@ class SubmissionRequestFinalizer( val accNo = previous.accNo val owner = previous.owner - fun deleteRemainingFiles(allFiles: Sequence): Sequence { + fun deleteRemainingFiles(allFiles: Flow): Flow { return allFiles .filter { subFiles.contains(it.filePath).not() || it.storageMode != current?.storageMode } - .onEachIndexed { i, file -> logger.info { "$accNo $owner Deleting file $i, path='${file.filePath}'" } } + .withIndex() + .onEach { (i, file) -> logger.info { "$accNo $owner Deleting file $i, path='${file.filePath}'" } } + .map { it.value } } logger.info { "$accNo ${previous.owner} Started deleting remaining submission files" } @@ -50,10 +58,10 @@ class SubmissionRequestFinalizer( logger.info { "$accNo ${previous.owner} Finished deleting remaining submission files" } } - private fun subFilesSet(sub: ExtSubmission?): Set { + private suspend fun subFilesSet(sub: ExtSubmission?): Set { return when (sub) { null -> emptySet() - else -> serializationService.fileSequence(sub).mapTo(mutableSetOf()) { it.filePath } + else -> serializationService.filesFlow(sub).map { it.filePath }.toSet() } } } diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestIndexer.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestIndexer.kt index 61681080ed..e668b41e30 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestIndexer.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestIndexer.kt @@ -4,10 +4,11 @@ import ac.uk.ebi.biostd.persistence.common.model.SubmissionRequestFile import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestFilesPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import ebi.ac.uk.extended.model.ExtSubmission -import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.withIndex import mu.KotlinLogging import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow import java.util.concurrent.atomic.AtomicInteger private val logger = KotlinLogging.logger {} @@ -36,9 +37,9 @@ class SubmissionRequestIndexer( private suspend fun indexSubmissionFiles(sub: ExtSubmission): Int { val elements = AtomicInteger(0) extSerializationService - .fileSequence(sub) - .mapIndexed { idx, file -> SubmissionRequestFile(sub.accNo, sub.version, idx + 1, file.filePath, file) } - .asFlow() + .filesFlow(sub) + .withIndex() + .map { (idx, file) -> SubmissionRequestFile(sub.accNo, sub.version, idx + 1, file.filePath, file) } .collect { logger.info { "${sub.accNo} ${sub.owner} Indexing submission file ${it.index}, path='${it.path}'" } filesRequestService.saveSubmissionRequestFile(it) diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestReleaser.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestReleaser.kt index 094cd93079..b41da85a76 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestReleaser.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestReleaser.kt @@ -14,11 +14,13 @@ import ebi.ac.uk.extended.model.FireFile import ebi.ac.uk.extended.model.NfsFile import kotlinx.coroutines.async import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.collectIndexed +import kotlinx.coroutines.flow.filterNot import kotlinx.coroutines.flow.map import kotlinx.coroutines.supervisorScope import mu.KotlinLogging import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow private val logger = KotlinLogging.logger {} @@ -99,9 +101,9 @@ class SubmissionRequestReleaser( private suspend fun releaseSubmission(sub: ExtSubmission) { logger.info { "${sub.accNo} ${sub.owner} Started releasing submission files over ${sub.storageMode}" } - serializationService.fileSequence(sub) + serializationService.filesFlow(sub) .filterNot { it is FireFile && it.published } - .forEachIndexed { idx, file -> releaseFile(sub, idx, file) } + .collectIndexed { idx, file -> releaseFile(sub, idx, file) } persistenceService.setAsReleased(sub.accNo) logger.info { "${sub.accNo} ${sub.owner} Finished releasing submission files over ${sub.storageMode}" } } diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/validator/filelist/FileListValidator.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/validator/filelist/FileListValidator.kt index 3f79f87f4e..7b56ea4872 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/validator/filelist/FileListValidator.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/validator/filelist/FileListValidator.kt @@ -13,8 +13,8 @@ import ebi.ac.uk.model.BioFile import ebi.ac.uk.model.constants.FileFields.FILE_TYPE import ebi.ac.uk.security.integration.model.api.SecurityUser import ebi.ac.uk.util.collections.ifNotEmpty -import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.onEmpty import kotlinx.coroutines.flow.take import kotlinx.coroutines.flow.toList import java.io.InputStream @@ -56,9 +56,8 @@ class FileListValidator( filesSource: FileSourcesList, ) { serializationService - .deserializeFileList(stream, format) - .ifEmpty { throw InvalidFileListException.emptyFileList(name) } - .asFlow() + .deserializeFileListAsFlow(stream, format) + .onEmpty { throw InvalidFileListException.emptyFileList(name) } .filter { filesSource.findExtFile(it.path, FILE_TYPE.value, it.attributes) == null } .take(FILE_LIST_LIMIT) .toList() diff --git a/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt b/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt index 23d3c71032..7032715772 100644 --- a/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt +++ b/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestCleanerTest.kt @@ -26,7 +26,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow @OptIn(ExperimentalCoroutinesApi::class) @ExtendWith(MockKExtension::class) @@ -96,7 +96,7 @@ class SubmissionRequestCleanerTest( every { loadedRequest.withNewStatus(CLEANED) } returns cleanedRequest coEvery { queryService.findExtByAccNo("S-BSST1", true) } returns current coEvery { requestService.getLoadedRequest("S-BSST1", 2) } returns loadedRequest - every { serializationService.fileSequence(current) } returns sequenceOf(currentFile) + every { serializationService.filesFlow(current) } returns flowOf(currentFile) coEvery { requestService.saveSubmissionRequest(cleanedRequest) } returns ("S-BSST1" to 2) coEvery { storageService.deleteSubmissionFile(current, currentFile) } answers { nothing } every { filesRequestService.getSubmissionRequestFiles("S-BSST1", 2, 0) } returns flowOf(requestFile) diff --git a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionRefreshApiTest.kt b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionRefreshApiTest.kt index 5cf838b355..2af0342e8f 100644 --- a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionRefreshApiTest.kt +++ b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionRefreshApiTest.kt @@ -199,7 +199,7 @@ class SubmissionRefreshApiTest( webClient.refreshSubmission(accNo) val files = fileListRepository - .findAllBySubmissionAccNoAndSubmissionVersionAndFileListName(accNo, 1, FILE_LIST_NAME) + .findByFileList(accNo, 1, FILE_LIST_NAME) .toList() assertThat(files).hasSize(1) diff --git a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionStorageModeTest.kt b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionStorageModeTest.kt index 6448d00499..007b3ab07b 100644 --- a/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionStorageModeTest.kt +++ b/submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionStorageModeTest.kt @@ -29,6 +29,7 @@ import ebi.ac.uk.extended.model.StorageMode.FIRE import ebi.ac.uk.extended.model.StorageMode.NFS import ebi.ac.uk.io.ext.createFile import ebi.ac.uk.io.ext.listFilesOrEmpty +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeAll @@ -177,10 +178,10 @@ class SubmissionStorageModeTest( assertThat(file).isInstanceOf(expectType.java) } - private fun assertFileListFile(fileList: ExtFileList, expectType: KClass<*>) { + private suspend fun assertFileListFile(fileList: ExtFileList, expectType: KClass<*>) { assertThat(fileList.fileName).isEqualTo("file-list") - val files = fileList.file.inputStream().use { serializationService.deserializeList(it).toList() } + val files = fileList.file.inputStream().use { serializationService.deserializeListAsFlow(it).toList() } assertThat(files).hasSize(1) val file = files.first() diff --git a/submission/submission-webapp/src/itest/resources/application.yml b/submission/submission-webapp/src/itest/resources/application.yml index 25ab5339ba..6ed8bb91ae 100644 --- a/submission/submission-webapp/src/itest/resources/application.yml +++ b/submission/submission-webapp/src/itest/resources/application.yml @@ -11,11 +11,16 @@ spring: physical-strategy: org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl use-new-id-generator-mappings: false + servlet: multipart: max-file-size: 10240MB max-request-size: 10240MB + mvc: + async: + request-timeout: -1 + rabbitmq: username: guest password: guest diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/RetryHandler.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/RetryHandler.kt index 8f115fae15..3f7de576a3 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/RetryHandler.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/RetryHandler.kt @@ -3,9 +3,6 @@ package ac.uk.ebi.biostd.submission.domain.service import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import kotlinx.coroutines.runBlocking import mu.KotlinLogging -import org.springframework.boot.context.event.ApplicationReadyEvent -import org.springframework.context.event.EventListener -import org.springframework.scheduling.annotation.Scheduled import java.time.Duration import java.time.temporal.ChronoUnit @@ -20,14 +17,14 @@ class RetryHandler( private val requestService: SubmissionRequestPersistenceService, ) { - @EventListener(ApplicationReadyEvent::class) + //@EventListener(ApplicationReadyEvent::class) fun onStart() = runBlocking { logger.info { "Re processing pending submission on application start" } requestService.getProcessingRequests(Duration.of(3, ChronoUnit.HOURS)) .collect { (accNo, version) -> reTriggerSafely(accNo, version) } } - @Scheduled(cron = "0 0 */3 * * ?") + //@Scheduled(cron = "0 0 */3 * * ?") fun onSchedule() = runBlocking { logger.info { "Scheduled re processing of pending submission" } requestService.getProcessingRequests(Duration.of(3, ChronoUnit.HOURS)) diff --git a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/stats/domain/service/SubmissionStatsServiceTest.kt b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/stats/domain/service/SubmissionStatsServiceTest.kt index 87d0b06134..4f594b7412 100644 --- a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/stats/domain/service/SubmissionStatsServiceTest.kt +++ b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/stats/domain/service/SubmissionStatsServiceTest.kt @@ -29,7 +29,7 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.springframework.web.multipart.MultipartFile import uk.ac.ebi.extended.serialization.service.ExtSerializationService -import uk.ac.ebi.extended.serialization.service.fileSequence +import uk.ac.ebi.extended.serialization.service.filesFlow import java.io.File @ExtendWith(MockKExtension::class) @@ -147,7 +147,7 @@ class SubmissionStatsServiceTest( every { file2.size } returns 3L every { submission.accNo } returns "S-BIAD123" coEvery { submissionStatsService.save(capture(savedStatSlot)) } returns stat - every { serializationService.fileSequence(submission) } returns sequenceOf(file1, file2) + every { serializationService.filesFlow(submission) } returns flowOf(file1, file2) coEvery { queryService.getExtByAccNo("S-BIAD123", includeFileListFiles = true) } returns submission val result = testInstance.calculateSubFilesSize("S-BIAD123") diff --git a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionDraftServiceTest.kt b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionDraftServiceTest.kt index d289835322..af3ace8b18 100644 --- a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionDraftServiceTest.kt +++ b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionDraftServiceTest.kt @@ -98,7 +98,7 @@ class SubmissionDraftServiceTest( coEvery { submissionQueryService.getExtByAccNo(DRAFT_KEY) } returns submission coEvery { draftPersistenceService.findSubmissionDraft(USER_ID, DRAFT_KEY) } returns null coEvery { draftPersistenceService.createSubmissionDraft(USER_ID, DRAFT_KEY, DRAFT_CONTENT) } returns testDraft - every { + coEvery { serializationService.serializeSubmission(toSubmissionMapper.toSimpleSubmission(submission), JsonPretty) } returns DRAFT_CONTENT diff --git a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/ExtSubmissionSubmitterTest.kt b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/ExtSubmissionSubmitterTest.kt index d68979cec9..95e89f6e03 100644 --- a/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/ExtSubmissionSubmitterTest.kt +++ b/submission/submission-webapp/src/test/kotlin/ac/uk/ebi/biostd/submission/submitter/ExtSubmissionSubmitterTest.kt @@ -87,7 +87,7 @@ internal class ExtSubmissionSubmitterTest( val submissionRequestSlot = slot() coEvery { persistenceService.getNextVersion("S-TEST123") } returns 2 - every { pageTabService.generatePageTab(submission) } returns submission + coEvery { pageTabService.generatePageTab(submission) } returns submission coEvery { requestService.createSubmissionRequest(capture(submissionRequestSlot)) } returns ("S-TEST123" to 2) testInstance.createRequest(ExtSubmitRequest(submission, "user@test.org", "TMP_123"))