Pivotal ID # 186322997: S-BIAD877 could not be loaded

buildSrc/src/main/kotlin/Dependencies.kt

@@ -25,6 +25,7 @@ import Versions.H2Version
 import Versions.JSONOrgVersion
 import Versions.JacksonVersion
 import Versions.JavaValidationApiVersion
+import Versions.JetBrainsAnnotationsVersion
 import Versions.JschVersion
 import Versions.JwtVersion
 import Versions.KMongoCoroutineVersion
@@ -74,6 +75,7 @@ object Versions {
     const val SpringVersion = "5.2.20.RELEASE"
     const val SpringAdminVersion = "2.3.1"
     const val KotlinVersion = "1.6.10"
+    const val JetBrainsAnnotationsVersion = "24.0.1"
     const val KotlinCoroutinesVersion = "1.6.3"
 
     const val KotlinLoggingVersion = "1.6.20"
@@ -194,6 +196,7 @@ object Dependencies {
     const val ZipUtil = "org.zeroturnaround:zt-zip:$ZipUtilVersion"
 
     // Kotlin specific
+    const val JetBrainsAnnotations = "org.jetbrains:annotations:$JetBrainsAnnotationsVersion"
     const val KotlinStdLib = "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$KotlinVersion"
     const val KotlinReflect = "org.jetbrains.kotlin:kotlin-reflect:$KotlinVersion"
     const val KotlinCoroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-core:$KotlinCoroutinesVersion"

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtFileListMapper.kt

@@ -8,7 +8,6 @@ import ebi.ac.uk.io.sources.FileSourcesList
 import ebi.ac.uk.io.use
 import ebi.ac.uk.model.FileList
 import ebi.ac.uk.model.canonicalName
-import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.onEach
 import mu.KotlinLogging
@@ -46,8 +45,7 @@ class ToExtFileListMapper(
             target: OutputStream,
         ) {
             val idx = AtomicInteger(0)
-            val sourceFiles = serializationService.deserializeFileList(input, format)
-                .asFlow()
+            val sourceFiles = serializationService.deserializeFileListAsFlow(input, format)
                 .onEach { file -> logger.info { "$accNo, Mapping file ${idx.getAndIncrement()}, path='${file.path}'" } }
                 .map { sources.getExtFile(it.path, it.type, it.attributes) }
             val files = extSerializationService.serialize(sourceFiles, target)

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapper.kt

@@ -7,6 +7,7 @@ import ebi.ac.uk.extended.model.ExtFileList
 import ebi.ac.uk.io.use
 import ebi.ac.uk.model.FileList
 import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
 import kotlinx.coroutines.flow.map
 import uk.ac.ebi.extended.serialization.service.ExtSerializationService
 import uk.ac.ebi.serialization.common.FilesResolver
@@ -20,56 +21,41 @@ class ToFileListMapper(
     private val extSerializationService: ExtSerializationService,
     private val filesResolver: FilesResolver,
 ) {
-    fun convert(fileList: ExtFileList): FileList = FileList(fileList.filePath, emptyFile(fileList.fileName))
+    suspend fun convert(fileList: ExtFileList): FileList = FileList(fileList.filePath, emptyFile(fileList.fileName))
 
-    fun serialize(fileList: ExtFileList, targetFormat: SubFormat, file: File): File {
+    suspend fun serialize(fileList: ExtFileList, targetFormat: SubFormat, file: File): File {
         toFile(fileList.file, targetFormat, file)
         return file
     }
 
-    fun serialize(fileListFiles: Sequence<ExtFile>, targetFormat: SubFormat, file: File): File {
-        toFile(fileListFiles, targetFormat, file)
-        return file
-    }
-
     suspend fun serialize(fileListFiles: Flow<ExtFile>, targetFormat: SubFormat, file: File): File {
         toFile(fileListFiles, targetFormat, file)
         return file
     }
 
-    private fun emptyFile(fileName: String): File {
+    private suspend fun emptyFile(fileName: String): File {
         val targetFile = filesResolver.createEmptyFile(fileName = fileName)
-        targetFile.outputStream().use { serializationService.serializeFileList(emptySequence(), SubFormat.JSON, it) }
+        targetFile.outputStream().use { serializationService.serializeFileList(emptyFlow(), SubFormat.JSON, it) }
         return targetFile
     }
 
-    private fun toFile(source: File, targetFormat: SubFormat, target: File): File {
+    private suspend fun toFile(source: File, targetFormat: SubFormat, target: File): File {
         use(source.inputStream(), target.outputStream()) { input, output -> copy(input, targetFormat, output) }
         return target
     }
 
-    private fun toFile(source: Sequence<ExtFile>, targetFormat: SubFormat, target: File): File {
-        target.outputStream().use { copy(source, targetFormat, it) }
-        return target
-    }
-
     private suspend fun toFile(source: Flow<ExtFile>, targetFormat: SubFormat, target: File): File {
         target.outputStream().use { copy(source, targetFormat, it) }
         return target
     }
 
-    private fun copy(source: Sequence<ExtFile>, targetFormat: SubFormat, target: OutputStream) {
-        val sourceFiles = source.map { it.toFile() }
-        serializationService.serializeFileList(sourceFiles, targetFormat, target)
-    }
-
     private suspend fun copy(source: Flow<ExtFile>, targetFormat: SubFormat, target: OutputStream) {
         val sourceFiles = source.map { it.toFile() }
         serializationService.serializeFileList(sourceFiles, targetFormat, target)
     }
 
-    private fun copy(input: InputStream, targetFormat: SubFormat, target: OutputStream) {
-        val sourceFiles = extSerializationService.deserializeList(input).map { it.toFile() }
+    private suspend fun copy(input: InputStream, targetFormat: SubFormat, target: OutputStream) {
+        val sourceFiles = extSerializationService.deserializeListAsFlow(input).map { it.toFile() }
         serializationService.serializeFileList(sourceFiles, targetFormat, target)
     }
 }

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapper.kt

@@ -6,7 +6,7 @@ import ebi.ac.uk.model.Section
 import ebi.ac.uk.model.SectionsTable
 
 class ToSectionMapper(private val toFileListMapper: ToFileListMapper) {
-    fun convert(sec: ExtSection): Section = Section(
+    suspend fun convert(sec: ExtSection): Section = Section(
         type = sec.type,
         accNo = sec.accNo,
         fileList = sec.fileList?.let { toFileListMapper.convert(it) },
@@ -16,6 +16,6 @@ class ToSectionMapper(private val toFileListMapper: ToFileListMapper) {
         sections = sec.sections.mapTo(mutableListOf()) { either -> either.bimap({ convert(it) }, { toTable(it) }) }
     )
 
-    private fun toTable(extSectionTable: ExtSectionTable): SectionsTable =
+    private suspend fun toTable(extSectionTable: ExtSectionTable): SectionsTable =
         SectionsTable(extSectionTable.sections.map { section -> convert(section) })
 }

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapper.kt

@@ -16,7 +16,7 @@ class ToSubmissionMapper(private val toSectionMapper: ToSectionMapper) {
      * Return a simple submission which does not contain submission secret information. Used to generate public
      * submission tab files.
      */
-    fun toSimpleSubmission(sub: ExtSubmission): Submission = Submission(
+    suspend fun toSimpleSubmission(sub: ExtSubmission): Submission = Submission(
         accNo = sub.accNo,
         section = toSectionMapper.convert(sub.section),
         attributes = sub.simpleAttributes(),

commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToFileListMapperTest.kt

@@ -7,10 +7,15 @@ import ebi.ac.uk.extended.model.ExtFileList
 import ebi.ac.uk.model.BioFile
 import io.github.glytching.junit.extension.folder.TemporaryFolder
 import io.github.glytching.junit.extension.folder.TemporaryFolderExtension
+import io.mockk.coEvery
 import io.mockk.every
 import io.mockk.impl.annotations.MockK
 import io.mockk.junit5.MockKExtension
 import io.mockk.mockkStatic
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.test.runTest
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
@@ -31,13 +36,13 @@ internal class ToFileListMapperTest(
     private val testInstance = ToFileListMapper(serializationService, extSerializationService, filesResolver)
 
     @Test
-    fun convert() {
+    fun convert() = runTest {
         every { filesResolver.createEmptyFile("fileList") } returns temporaryFolder.createFile("target-file-list.json")
-        every { serializationService.serializeFileList(any<Sequence<BioFile>>(), any(), any()) } answers {
-            val sequence = arg<Sequence<BioFile>>(0)
+        coEvery { serializationService.serializeFileList(any<Flow<BioFile>>(), any(), any()) } coAnswers {
+            val flow = arg<Flow<BioFile>>(0)
             val format = arg<SubFormat>(1)
             val stream = arg<OutputStream>(2)
-            stream.use { it.write("${sequence.toList().size}-$format".toByteArray()) }
+            stream.use { it.write("${flow.toList().size}-$format".toByteArray()) }
         }
 
         val fileList = testInstance.convert(extFileList)
@@ -47,16 +52,16 @@ internal class ToFileListMapperTest(
     }
 
     @Test
-    fun serialize() {
+    fun serialize() = runTest {
         val target = temporaryFolder.createFile("target-file-list.json")
         mockkStatic(TO_FILE_EXTENSIONS)
         every { extFile.toFile() } returns bioFile
-        every { extSerializationService.deserializeList(any()) } returns sequenceOf(extFile)
-        every { serializationService.serializeFileList(any<Sequence<BioFile>>(), any(), any()) } answers {
-            val sequence = arg<Sequence<BioFile>>(0)
+        coEvery { extSerializationService.deserializeListAsFlow(any()) } returns flowOf(extFile)
+        coEvery { serializationService.serializeFileList(any<Flow<BioFile>>(), any(), any()) } coAnswers {
+            val flow = arg<Flow<BioFile>>(0)
             val format = arg<SubFormat>(1)
             val stream = arg<OutputStream>(2)
-            stream.use { it.write("${sequence.toList().size}-$format".toByteArray()) }
+            stream.use { it.write("${flow.toList().size}-$format".toByteArray()) }
         }
 
         val result = testInstance.serialize(extFileList, SubFormat.XML, target)

commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSectionMapperTest.kt

@@ -18,11 +18,13 @@ import ebi.ac.uk.model.Link
 import ebi.ac.uk.model.LinksTable
 import ebi.ac.uk.model.Section
 import ebi.ac.uk.util.collections.second
+import io.mockk.coEvery
 import io.mockk.every
 import io.mockk.impl.annotations.MockK
 import io.mockk.junit5.MockKExtension
 import io.mockk.mockk
 import io.mockk.mockkStatic
+import kotlinx.coroutines.test.runTest
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
@@ -41,7 +43,7 @@ class ToSectionMapperTest(
     @MockK val extFile: ExtFile,
     @MockK val extFileTable: ExtFileTable,
     @MockK val extLink: ExtLink,
-    @MockK val extLinkTable: ExtLinkTable
+    @MockK val extLinkTable: ExtLinkTable,
 ) {
     private val subExtSection = ExtSection(type = "subtype", accNo = "accNo1")
     private val subSection = Section(type = "subtype", accNo = "accNo1")
@@ -58,7 +60,7 @@ class ToSectionMapperTest(
     private val testInstance = ToSectionMapper(toFileListMapper)
 
     @Test
-    fun toSection() {
+    fun toSection() = runTest {
         mockkStatic(
             TO_ATTRIBUTE_EXTENSIONS,
             TO_FILE_EXTENSIONS,
@@ -67,7 +69,7 @@ class ToSectionMapperTest(
         ) {
             every { extAttribute.toAttribute() } returns attribute
             every { extFile.toFile() } returns file
-            every { toFileListMapper.convert(extFileList) } returns fileList
+            coEvery { toFileListMapper.convert(extFileList) } returns fileList
             every { extLink.toLink() } returns link
             every { extFileTable.toTable() } returns fileTable
             every { extLinkTable.toTable() } returns linkTable

commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionMapperTest.kt

@@ -11,9 +11,10 @@ import ebi.ac.uk.model.extensions.releaseDate
 import ebi.ac.uk.model.extensions.rootPath
 import ebi.ac.uk.model.extensions.title
 import ebi.ac.uk.test.basicExtSubmission
-import io.mockk.every
+import io.mockk.coEvery
 import io.mockk.impl.annotations.MockK
 import io.mockk.junit5.MockKExtension
+import kotlinx.coroutines.test.runTest
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
@@ -37,9 +38,9 @@ class ToSubmissionMapperTest(
     private val testInstance = ToSubmissionMapper(toSectionMapper)
 
     @Test
-    fun toSimpleSubmission() {
+    fun toSimpleSubmission() = runTest {
         val section = Section()
-        every { toSectionMapper.convert(extSubmission.section) } returns section
+        coEvery { toSectionMapper.convert(extSubmission.section) } returns section
 
         val submission = testInstance.toSimpleSubmission(extSubmission)
 

commons/commons-model-extended-serialization/build.gradle.kts

@@ -36,4 +36,5 @@ dependencies {
     BaseTestRuntimeDependencies.forEach { testImplementation(it) }
 
     testFixturesApi(Arrow)
+    testFixturesImplementation(KotlinCoroutines)
 }

commons/commons-model-extended-serialization/gradle.properties

@@ -1 +1 @@
-coverage=0.81
+coverage=0.78

commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationService.kt

@@ -39,7 +39,8 @@ import uk.ac.ebi.extended.serialization.serializers.ExtSectionSerializer
 import uk.ac.ebi.extended.serialization.serializers.ExtSectionsTableSerializer
 import uk.ac.ebi.extended.serialization.serializers.ExtSubmissionSerializer
 import uk.ac.ebi.extended.serialization.serializers.OffsetDateTimeSerializer
-import uk.ac.ebi.serialization.extensions.deserializeList
+import uk.ac.ebi.serialization.extensions.deserializeAsFlow
+import uk.ac.ebi.serialization.extensions.deserializeAsSequence
 import uk.ac.ebi.serialization.extensions.serializeFlow
 import uk.ac.ebi.serialization.extensions.serializeList
 import uk.ac.ebi.serialization.serializers.EitherSerializer
@@ -54,16 +55,17 @@ data class Properties(val includeFileListFiles: Boolean) : StringWriter()
 class ExtSerializationService private constructor(val mapper: ObjectMapper) {
     fun serialize(sub: ExtSubmission, props: Properties = Properties(false)): String = serializeElement(sub, props)
     fun serialize(files: Sequence<ExtFile>, stream: OutputStream): Int = mapper.serializeList(files, stream)
-    suspend fun serialize(files: Flow<ExtFile>, stream: OutputStream): Int = mapper.serializeFlow(files, stream)
     fun serialize(file: ExtFile): String = serializeElement(file)
     fun serialize(table: ExtFileTable): String = serializeElement(table)
     fun serialize(extPage: WebExtPage): String = serializeElement(extPage)
+    suspend fun serialize(files: Flow<ExtFile>, stream: OutputStream): Int = mapper.serializeFlow(files, stream)
 
     fun deserialize(value: String): ExtSubmission = mapper.readValue(value)
     fun deserializeFile(value: String): ExtFile = mapper.readValue(value)
-    fun deserializeList(stream: InputStream): Sequence<ExtFile> = mapper.deserializeList(stream)
     fun deserializePage(value: String): ExtPage = mapper.readValue(value)
     fun deserializeTable(value: String): ExtFileTable = mapper.readValue(value)
+    fun deserializeListAsSequence(stream: InputStream): Sequence<ExtFile> = mapper.deserializeAsSequence(stream)
+    suspend fun deserializeListAsFlow(stream: InputStream): Flow<ExtFile> = mapper.deserializeAsFlow(stream)
 
     /**
      * Serialize a generic element. ONLY for testing purpose.

commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt

@@ -4,19 +4,21 @@ import ebi.ac.uk.extended.model.ExtFile
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.extended.model.allFileList
 import ebi.ac.uk.extended.model.allSectionsFiles
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emitAll
+import kotlinx.coroutines.flow.flow
 
 /**
- * Return a sequence with all the files of a submission. Pagetab files are retrieved first, followed by section files
+ * Return a flow with all the files of a submission. Pagetab files are retrieved first, followed by section files
  * and file list files.
  */
-fun ExtSerializationService.fileSequence(submission: ExtSubmission): Sequence<ExtFile> {
-    return sequence {
-        submission.pageTabFiles.forEach { yield(it) }
-        submission.allFileList.forEach { fileList -> fileList.pageTabFiles.forEach { yield(it) } }
-
-        submission.allSectionsFiles.forEach { yield(it) }
+fun ExtSerializationService.filesFlow(submission: ExtSubmission): Flow<ExtFile> {
+    return flow {
+        submission.pageTabFiles.forEach { emit(it) }
+        submission.allFileList.forEach { fileList -> fileList.pageTabFiles.forEach { emit(it) } }
+        submission.allSectionsFiles.forEach { emit(it) }
         submission.allFileList
             .map { it.file }
-            .forEach { it.inputStream().use { stream -> deserializeList(stream).forEach { file -> yield(file) } } }
+            .forEach { it.inputStream().use { stream -> emitAll(deserializeListAsFlow(stream)) } }
     }
 }

commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/FileProcessingService.kt

@@ -9,7 +9,6 @@ import ebi.ac.uk.extended.model.ExtSectionTable
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.io.use
 import ebi.ac.uk.util.collections.mapLeft
-import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.map
 import uk.ac.ebi.serialization.common.FilesResolver
 import java.io.File
@@ -95,7 +94,7 @@ class FileProcessingService(
             inputFile.inputStream(),
             outputFile.outputStream()
         ) { input, output ->
-            val files = serializationService.deserializeList(input).asFlow()
+            val files = serializationService.deserializeListAsFlow(input)
             serializationService.serialize(files.map { processFile(it) }, output)
         }
 

commons/commons-model-extended-serialization/src/test/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExtTest.kt

@@ -15,6 +15,8 @@ import ebi.ac.uk.extended.model.FireFile
 import ebi.ac.uk.extended.model.StorageMode
 import io.github.glytching.junit.extension.folder.TemporaryFolder
 import io.github.glytching.junit.extension.folder.TemporaryFolderExtension
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.test.runTest
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
@@ -28,7 +30,7 @@ internal class ExtSerializationServiceExtTest(
     private val testInstance: ExtSerializationService = ExtSerializationService()
 
     @Test
-    fun fileSequence() {
+    fun fileFlow() = runTest {
         val fileList1 = tmpFolder.createFile("f1.json")
         val files = (4..1000).map { createFireFile(it) }
 
@@ -39,7 +41,7 @@ internal class ExtSerializationServiceExtTest(
         val submission = createTestSubmission(fileList1, pageTabFile, sectionFile, sectionTableFile)
         val filesCount = testInstance.serialize(files.asSequence(), fileList1.outputStream())
 
-        val result = testInstance.fileSequence(submission).toList()
+        val result = testInstance.filesFlow(submission).toList()
         assertThat(filesCount).isEqualTo(files.size)
         assertThat(result).containsExactlyInAnyOrder(pageTabFile, sectionFile, sectionTableFile, *files.toTypedArray())
     }