Pivotal ID # 176377632: File list extension is missing in BIA studies

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/constants/Fields.kt

@@ -4,12 +4,15 @@ private const val ACC_NO_FIELD = "accno"
 private const val SECTION_FIELD = "section"
 private const val ATTRIBUTES_FIELD = "attributes"
 
-val RESERVED_ATTRIBUTES = setOf(
+val SUBMISSION_RESERVED_ATTRIBUTES = setOf(
     SubFields.RELEASE_DATE.value,
     SubFields.TITLE.value,
     SubFields.ATTACH_TO.value,
     SubFields.ON_BEHALF.value,
-    SubFields.ROOT_PATH.value)
+    SubFields.ROOT_PATH.value
+)
+
+val SECTION_RESERVED_ATTRIBUTES = setOf(SectionFields.FILE_LIST.value)
 
 interface Fields {
     val value: String?

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/from/ToExtSection.kt

@@ -3,16 +3,16 @@ package ebi.ac.uk.extended.mapping.from
 import ebi.ac.uk.extended.model.ExtSection
 import ebi.ac.uk.io.sources.FilesSource
 import ebi.ac.uk.model.Section
+import ebi.ac.uk.model.constants.SECTION_RESERVED_ATTRIBUTES
 
 internal const val TO_EXT_SECTION_EXTENSIONS = "ebi.ac.uk.extended.mapping.from.ToExtSectionKt"
 
-fun Section.toExtSection(source: FilesSource): ExtSection {
-    return ExtSection(
-        type = type,
-        accNo = accNo,
-        fileList = fileList?.toExtFileList(source),
-        attributes = attributes.map { it.toExtAttribute() },
-        files = files.map { either -> either.bimap({ it.toExtFile(source) }, { it.toExtTable(source) }) },
-        links = links.map { either -> either.bimap({ it.toExtLink() }, { it.toExtTable() }) },
-        sections = sections.map { either -> either.bimap({ it.toExtSection(source) }, { it.toExtTable(source) }) })
-}
+fun Section.toExtSection(source: FilesSource): ExtSection = ExtSection(
+    type = type,
+    accNo = accNo,
+    fileList = fileList?.toExtFileList(source),
+    attributes = attributes.filterNot { SECTION_RESERVED_ATTRIBUTES.contains(it.name) }.map { it.toExtAttribute() },
+    files = files.map { either -> either.bimap({ it.toExtFile(source) }, { it.toExtTable(source) }) },
+    links = links.map { either -> either.bimap({ it.toExtLink() }, { it.toExtTable() }) },
+    sections = sections.map { either -> either.bimap({ it.toExtSection(source) }, { it.toExtTable(source) }) }
+)

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSection.kt

@@ -5,13 +5,12 @@ import ebi.ac.uk.model.Section
 
 internal const val TO_SECTION_EXTENSIONS = "ebi.ac.uk.extended.mapping.to.ToSectionKt"
 
-fun ExtSection.toSection(): Section {
-    return Section(
-        type = type,
-        accNo = accNo,
-        fileList = fileList?.toFileList(),
-        attributes = attributes.mapTo(mutableListOf()) { it.toAttribute() },
-        files = files.mapTo(mutableListOf()) { either -> either.bimap({ it.toFile() }, { it.toTable() }) },
-        links = links.mapTo(mutableListOf()) { either -> either.bimap({ it.toLink() }, { it.toTable() }) },
-        sections = sections.mapTo(mutableListOf()) { either -> either.bimap({ it.toSection() }, { it.toTable() }) })
-}
+fun ExtSection.toSection(): Section = Section(
+    type = type,
+    accNo = accNo,
+    fileList = fileList?.toFileList(),
+    attributes = attributes.mapTo(mutableListOf()) { it.toAttribute() },
+    files = files.mapTo(mutableListOf()) { either -> either.bimap({ it.toFile() }, { it.toTable() }) },
+    links = links.mapTo(mutableListOf()) { either -> either.bimap({ it.toLink() }, { it.toTable() }) },
+    sections = sections.mapTo(mutableListOf()) { either -> either.bimap({ it.toSection() }, { it.toTable() }) }
+)

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmission.kt

@@ -6,46 +6,25 @@ import ebi.ac.uk.model.Submission
 import ebi.ac.uk.model.constants.SubFields
 import ebi.ac.uk.model.constants.SubFields.ATTACH_TO
 import ebi.ac.uk.model.constants.SubFields.PUBLIC_ACCESS_TAG
-import ebi.ac.uk.model.extensions.creationTime
-import ebi.ac.uk.model.extensions.modificationTime
 import ebi.ac.uk.model.extensions.releaseTime
-import ebi.ac.uk.model.extensions.secretKey
 
 /**
  * Return a simple submission which does not contain submission secret information.
  */
-fun ExtSubmission.toSimpleSubmission(): Submission {
-    return Submission(
-        accNo = accNo,
-        section = section.toSection(),
-        attributes = getSubmissionAttributes(),
-        tags = tags.mapTo(mutableListOf()) { Pair(it.name, it.value) }
-    )
-}
+fun ExtSubmission.toSimpleSubmission(): Submission = Submission(
+    accNo = accNo,
+    section = section.toSection(),
+    attributes = getSubmissionAttributes(),
+    tags = tags.mapTo(mutableListOf()) { Pair(it.name, it.value) }
+)
 
 private fun ExtSubmission.getSubmissionAttributes(): List<Attribute> {
     val subAttrs = attributes.map { it.toAttribute() }.toMutableSet()
+
     title?.let { subAttrs.add(Attribute(SubFields.TITLE, it)) }
     releaseTime?.let { subAttrs.add(Attribute(SubFields.RELEASE_DATE, it.toLocalDate())) }
     rootPath?.let { subAttrs.add(Attribute(SubFields.ROOT_PATH, it)) }
     projects.filter { it.accNo != PUBLIC_ACCESS_TAG.value }.forEach { subAttrs.add(Attribute(ATTACH_TO, it.accNo)) }
-    return subAttrs.toList()
-}
 
-/**
- * Return a extended submission which contains submission secret information and internal information.
- */
-fun ExtSubmission.toExtSubmission(): Submission {
-    val submission = Submission(
-        accNo = accNo,
-        section = section.toSection(),
-        attributes = attributes.map { it.toAttribute() }.toMutableList(),
-        tags = tags.mapTo(mutableListOf()) { Pair(it.name, it.value) }
-    )
-
-    submission.secretKey = secretKey
-    submission.creationTime = creationTime
-    submission.modificationTime = modificationTime
-    submission.releaseTime = releaseTime
-    return submission
+    return subAttrs.toList()
 }

commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/from/ToExtSectionTest.kt

@@ -18,6 +18,7 @@ import ebi.ac.uk.model.Link
 import ebi.ac.uk.model.LinksTable
 import ebi.ac.uk.model.Section
 import ebi.ac.uk.model.SectionsTable
+import ebi.ac.uk.model.constants.SectionFields
 import ebi.ac.uk.util.collections.second
 import io.mockk.every
 import io.mockk.impl.annotations.MockK
@@ -39,6 +40,7 @@ class ToExtSectionTest(
     @MockK val sectionTable: SectionsTable,
     @MockK val extFileList: ExtFileList,
     @MockK val extAttribute: ExtAttribute,
+    @MockK val fileListAttribute: ExtAttribute,
     @MockK val extFile: ExtFile,
     @MockK val extFileTable: ExtFileTable,
     @MockK val extLink: ExtLink,
@@ -66,6 +68,8 @@ class ToExtSectionTest(
             TO_EXT_TABLE_EXTENSIONS,
             TO_EXT_SECTION_EXTENSIONS) {
 
+            every { attribute.name } returns "attr1"
+            every { fileListAttribute.name } returns SectionFields.FILE_LIST.value
             every { attribute.toExtAttribute() } returns extAttribute
             every { file.toExtFile(fileSource) } returns extFile
             every { fileList.toExtFileList(fileSource) } returns extFileList

commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/json/serialization/SectionJsonSerializer.kt

@@ -6,6 +6,7 @@ import ac.uk.ebi.biostd.json.common.writeObj
 import com.fasterxml.jackson.core.JsonGenerator
 import com.fasterxml.jackson.databind.SerializerProvider
 import com.fasterxml.jackson.databind.ser.std.StdSerializer
+import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.Section
 import ebi.ac.uk.model.constants.SectionFields
 
@@ -14,10 +15,15 @@ internal class SectionJsonSerializer : StdSerializer<Section>(Section::class.jav
         gen.writeObj {
             writeJsonString(SectionFields.ACC_NO, section.accNo)
             writeJsonString(SectionFields.TYPE, section.type)
-            writeJsonArray(SectionFields.ATTRIBUTES, section.attributes)
+            writeJsonArray(SectionFields.ATTRIBUTES, sectionAttributes(section))
             writeJsonArray(SectionFields.FILES, section.files)
             writeJsonArray(SectionFields.LINKS, section.links)
             writeJsonArray(SectionFields.SUBSECTIONS, section.sections)
         }
     }
+
+    private fun sectionAttributes(section: Section): List<Attribute> = when (val fileList = section.fileList) {
+        null -> section.attributes
+        else -> section.attributes.plus(Attribute(SectionFields.FILE_LIST.value, "${fileList.name}.json"))
+    }
 }

commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/tsv/serialization/TsvToStringSerializer.kt

@@ -1,6 +1,7 @@
 package ac.uk.ebi.biostd.tsv.serialization
 
 import ebi.ac.uk.base.isNotBlank
+import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.File
 import ebi.ac.uk.model.FilesTable
 import ebi.ac.uk.model.Header
@@ -10,6 +11,7 @@ import ebi.ac.uk.model.Section
 import ebi.ac.uk.model.SectionsTable
 import ebi.ac.uk.model.Submission
 import ebi.ac.uk.model.Table
+import ebi.ac.uk.model.constants.SectionFields.FILE_LIST
 import ebi.ac.uk.model.constants.TableFields.FILES_TABLE
 import ebi.ac.uk.model.constants.TableFields.LINKS_TABLE
 
@@ -42,7 +44,7 @@ class TsvToStringSerializer {
     private fun serializeSection(builder: TsvBuilder, section: Section, parentAccNo: String? = null) {
         builder.addSeparator()
         builder.addSecDescriptor(section.type, section.accNo, parentAccNo)
-        section.attributes.forEach(builder::addAttr)
+        sectionAttributes(section).forEach(builder::addAttr)
 
         section.links.forEach {
             either -> either.fold({ addLink(builder, it) }, { addTable(builder, it, LINKS_TABLE.toString()) }) }
@@ -55,6 +57,11 @@ class TsvToStringSerializer {
         }
     }
 
+    private fun sectionAttributes(section: Section): List<Attribute> = when (val fileList = section.fileList) {
+        null -> section.attributes
+        else -> section.attributes.plus(Attribute(FILE_LIST.value, "${fileList.name}.pagetab.tsv"))
+    }
+
     private fun getHeader(table: SectionsTable, parentAccNo: String? = null) =
         "${table.elements.first().type}[${if (parentAccNo.isNotBlank()) "$parentAccNo" else ""}]"
 

commons/commons-serialization/src/main/kotlin/ac/uk/ebi/biostd/xml/serializer/SectionSerializer.kt

@@ -6,21 +6,27 @@ import ac.uk.ebi.biostd.xml.common.writeXmlCollection
 import ac.uk.ebi.biostd.xml.common.writeXmlObj
 import com.fasterxml.jackson.databind.SerializerProvider
 import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator
+import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.Section
 import ebi.ac.uk.model.constants.SectionFields
+import ebi.ac.uk.model.constants.SectionFields.FILE_LIST
 
 class SectionSerializer : XmlStdSerializer<Section>(Section::class.java) {
-
     override fun serializeXml(value: Section, gen: ToXmlGenerator, provider: SerializerProvider) {
         with(gen) {
             writeXmlObj(SectionFields.SECTION) {
                 writeXmlAttr(SectionFields.ACC_NO, value.accNo)
                 writeXmlAttr(SectionFields.TYPE, value.type)
-                writeXmlCollection(SectionFields.ATTRIBUTES, value.attributes)
+                writeXmlCollection(SectionFields.ATTRIBUTES, sectionAttributes(value))
                 writeXmlCollection(SectionFields.LINKS, value.links)
                 writeXmlCollection(SectionFields.FILES, value.files)
                 writeXmlCollection(SectionFields.SUBSECTIONS, value.sections)
             }
         }
     }
+
+    private fun sectionAttributes(section: Section): List<Attribute> = when (val fileList = section.fileList) {
+        null -> section.attributes
+        else -> section.attributes.plus(Attribute(FILE_LIST.value, "${fileList.name}.xml"))
+    }
 }

commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/JsonSerializerTest.kt

@@ -9,7 +9,6 @@ import org.junit.jupiter.api.Test
  * TODO: add extended submission test
  */
 class JsonSerializerTest {
-
     private val testInstance = JsonSerializer.mapper
     private val submission = createVenousBloodMonocyte()
 

commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/json/serialization/SectionJsonSerializerTest.kt

@@ -0,0 +1,52 @@
+package ac.uk.ebi.biostd.json.serialization
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.module.SimpleModule
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import ebi.ac.uk.dsl.json.jsonArray
+import ebi.ac.uk.dsl.json.jsonObj
+import ebi.ac.uk.dsl.section
+import ebi.ac.uk.model.FileList
+import ebi.ac.uk.model.Section
+import org.junit.jupiter.api.Test
+import org.skyscreamer.jsonassert.JSONAssert.assertEquals
+import org.skyscreamer.jsonassert.JSONCompareMode.LENIENT
+
+class SectionJsonSerializerTest {
+    private val testInstance = createSerializer()
+    private val section = section("Study") {
+        accNo = "SECT-001"
+        fileList = FileList("file-list")
+    }
+
+    @Test
+    fun `serialize section`() {
+        val json = testInstance.writeValueAsString(section)
+        val expected = jsonObj {
+            "type" to "Study"
+            "accno" to "SECT-001"
+            "attributes" to jsonArray(jsonObj {
+                "name" to "File List"
+                "value" to "file-list.json"
+                "reference" to false
+            })
+        }
+
+        assertEquals("invalid submission json", json, expected.toString(), LENIENT)
+    }
+
+    companion object {
+        fun createSerializer(): ObjectMapper {
+            val module = SimpleModule()
+            module.addSerializer(Section::class.java, SectionJsonSerializer())
+
+            return jacksonObjectMapper().apply {
+                registerModule(module)
+                setSerializationInclusion(Include.NON_EMPTY)
+                configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+            }
+        }
+    }
+}

commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/tsv/TsvToStringSerializerTest.kt

@@ -10,6 +10,7 @@ import ebi.ac.uk.dsl.tsv
 import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.AttributeDetail
 import ebi.ac.uk.model.File
+import ebi.ac.uk.model.FileList
 import ebi.ac.uk.model.FilesTable
 import ebi.ac.uk.model.Link
 import ebi.ac.uk.model.LinksTable
@@ -39,12 +40,14 @@ class TsvToStringSerializerTest {
     @Test
     fun `serialize section`() {
         val section = section("Study") {
+            fileList = FileList("file-list")
             attribute("Project", "Test Project")
         }
 
         val expectedTsv = tsv {
             line("Study")
             line("Project", "Test Project")
+            line("File List", "file-list.pagetab.tsv")
             line()
         }.toString()
 

commons/commons-serialization/src/test/kotlin/ac/uk/ebi/biostd/xml/serializer/SectionSerializerTest.kt

@@ -7,19 +7,22 @@ import ac.uk.ebi.biostd.xml.XmlSerializer
 import ebi.ac.uk.dsl.file
 import ebi.ac.uk.dsl.link
 import ebi.ac.uk.dsl.section
+import ebi.ac.uk.model.FileList
+import ebi.ac.uk.model.constants.SectionFields.FILE_LIST
 import org.junit.jupiter.api.Test
 import org.redundent.kotlin.xml.xml
 import org.xmlunit.assertj.XmlAssert.assertThat
 
 private const val LINK_URL = "link_url"
 private const val ACC_NO = "acc no"
 private const val TYPE = "sec type"
+private const val FILE_LIST_NAME = "file-list"
 
 class SectionSerializerTest {
     private val testInstance = XmlSerializer.mapper
-
     private val section = section(TYPE) {
         accNo = ACC_NO
+        fileList = FileList(FILE_LIST_NAME)
         link(LINK_URL)
         file(FILE_NAME) {
             type = FILE_TYPE
@@ -33,6 +36,12 @@ class SectionSerializerTest {
         val expected = xml("section") {
             attribute("accno", ACC_NO)
             attribute("type", TYPE)
+            "attributes" {
+                "attribute" {
+                    "name" { -FILE_LIST.value }
+                    "value" { -"$FILE_LIST_NAME.xml" }
+                }
+            }
             "links" {
                 "link" {
                     "url" { -LINK_URL }