Pivotal ID # 177267343: Validate EU-ToxRisk Submissions

client/bio-commandline/src/test/kotlin/uk/ac/ebi/biostd/client/cli/commands/SubmitCommandTest.kt

@@ -13,7 +13,6 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.extension.ExtendWith
-import uk.ac.ebi.biostd.client.cli.commands.SubmitCommand
 import uk.ac.ebi.biostd.client.cli.services.SubmissionRequest
 import uk.ac.ebi.biostd.client.cli.services.SubmissionService
 

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/constants/Fields.kt

@@ -42,7 +42,8 @@ enum class SubFields(override val value: String) : Fields {
     SECRET("secretKey"),
     ATTACH_TO("AttachTo"),
     ON_BEHALF("onBehalf"),
-    ACC_NO_TEMPLATE("AccNoTemplate");
+    ACC_NO_TEMPLATE("AccNoTemplate"),
+    COLLECTION_VALIDATOR("CollectionValidator");
 
     override fun toString(): String {
         return value

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/extensions/FileExt.kt

@@ -8,3 +8,6 @@ var File.type: String
     set(value) {
         this[FileFields.TYPE] = value
     }
+
+val File.extension: String
+    get() = path.substringAfterLast(".")

commons/commons-bio/src/test/kotlin/ebi/ac/uk/model/extensions/FileExtTest.kt

@@ -16,4 +16,10 @@ class FileExtTest {
         assertThat(file.attributes).hasSize(1)
         assertThat(file.attributes.first()).isEqualTo(Attribute(FileFields.TYPE, "Text"))
     }
+
+    @Test
+    fun extension() {
+        val file = File("/a/path/to/File1.txt")
+        assertThat(file.extension).isEqualTo("txt")
+    }
 }

commons/commons-model-extended-mapping/src/main/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmission.kt

@@ -3,9 +3,12 @@ package ebi.ac.uk.extended.mapping.to
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.model.Attribute
 import ebi.ac.uk.model.Submission
-import ebi.ac.uk.model.constants.SubFields
 import ebi.ac.uk.model.constants.SubFields.ATTACH_TO
+import ebi.ac.uk.model.constants.SubFields.COLLECTION_VALIDATOR
 import ebi.ac.uk.model.constants.SubFields.PUBLIC_ACCESS_TAG
+import ebi.ac.uk.model.constants.SubFields.RELEASE_DATE
+import ebi.ac.uk.model.constants.SubFields.ROOT_PATH
+import ebi.ac.uk.model.constants.SubFields.TITLE
 
 /**
  * Return a simple submission which does not contain submission secret information.
@@ -18,12 +21,17 @@ fun ExtSubmission.toSimpleSubmission(): Submission = Submission(
 )
 
 private fun ExtSubmission.getSubmissionAttributes(): List<Attribute> {
-    val subAttrs = attributes.map { it.toAttribute() }.toMutableSet()
-
-    title?.let { subAttrs.add(Attribute(SubFields.TITLE, it)) }
-    releaseTime?.let { subAttrs.add(Attribute(SubFields.RELEASE_DATE, it.toLocalDate())) }
-    rootPath?.let { subAttrs.add(Attribute(SubFields.ROOT_PATH, it)) }
+    val subAttrs = pageTabAttributes
+    title?.let { subAttrs.add(Attribute(TITLE, it)) }
+    releaseTime?.let { subAttrs.add(Attribute(RELEASE_DATE, it.toLocalDate())) }
+    rootPath?.let { subAttrs.add(Attribute(ROOT_PATH, it)) }
     collections.filter { it.accNo != PUBLIC_ACCESS_TAG.value }.forEach { subAttrs.add(Attribute(ATTACH_TO, it.accNo)) }
 
     return subAttrs.toList()
 }
+
+private val ExtSubmission.pageTabAttributes: MutableSet<Attribute>
+    get() = attributes
+        .filterNot { it.name == COLLECTION_VALIDATOR.value }
+        .map { it.toAttribute() }
+        .toMutableSet()

commons/commons-model-extended-mapping/src/test/kotlin/ebi/ac/uk/extended/mapping/to/ToSubmissionTest.kt

@@ -0,0 +1,50 @@
+package ebi.ac.uk.extended.mapping.to
+
+import ebi.ac.uk.extended.model.ExtAttribute
+import ebi.ac.uk.extended.model.ExtCollection
+import ebi.ac.uk.model.Attribute
+import ebi.ac.uk.model.Submission
+import ebi.ac.uk.model.extensions.attachTo
+import ebi.ac.uk.model.extensions.releaseDate
+import ebi.ac.uk.model.extensions.rootPath
+import ebi.ac.uk.model.extensions.title
+import ebi.ac.uk.test.basicExtSubmission
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import java.time.OffsetDateTime
+import java.time.ZoneOffset.UTC
+
+class ToSubmissionTest {
+    private val extSubmission = basicExtSubmission.copy(
+        rootPath = "/a/root/path",
+        collections = listOf(ExtCollection("BioImages")),
+        releaseTime = OffsetDateTime.of(2019, 9, 21, 0, 0, 0, 0, UTC),
+        attributes = listOf(
+            ExtAttribute("Type", "Experiment"),
+            ExtAttribute("CollectionValidator", "BioImagesValidator")))
+
+    @Test
+    fun toSimpleSubmission() {
+        val submission = extSubmission.toSimpleSubmission()
+        assertThat(submission.accNo).isEqualTo("S-TEST123")
+        assertSection(submission)
+        assertSubmissionAttributes(submission)
+    }
+
+    private fun assertSection(submission: Submission) {
+        assertThat(submission.section.type).isEqualTo("Study")
+        assertThat(submission.section.attributes).isEmpty()
+        assertThat(submission.section.files).isEmpty()
+        assertThat(submission.section.links).isEmpty()
+        assertThat(submission.section.sections).isEmpty()
+    }
+
+    private fun assertSubmissionAttributes(submission: Submission) {
+        assertThat(submission.attributes).hasSize(5)
+        assertThat(submission.attributes).contains(Attribute("Type", "Experiment"))
+        assertThat(submission.title).isEqualTo("Test Submission")
+        assertThat(submission.attachTo).isEqualTo("BioImages")
+        assertThat(submission.releaseDate).isEqualTo("2019-09-21")
+        assertThat(submission.rootPath).isEqualTo("/a/root/path")
+    }
+}

commons/commons-model-extended/src/main/kotlin/ebi/ac/uk/extended/model/ExtSubmissionExtensions.kt

@@ -8,5 +8,8 @@ val ExtSubmission.allSections
 val ExtSubmission.allFileList
     get(): List<ExtFileList> = allSections.mapNotNull { it.fileList }
 
+val ExtSubmission.allFiles
+    get(): List<ExtFile> = allSections.flatMap { it.allFiles }
+
 val ExtSubmission.isCollection
     get(): Boolean = section.type == PROJECT_TYPE

commons/commons-test/build.gradle.kts

@@ -7,6 +7,7 @@ import TestDependencies.JunitExtensions
 
 dependencies {
     api(project(":commons:commons-bio"))
+    api(project(":commons:commons-model-extended"))
 
     implementation(Arrow)
     implementation(AssertJ)

commons/commons-test/src/main/kotlin/ebi/ac/uk/test/ExtSubmissionFactory.kt

@@ -0,0 +1,29 @@
+package ebi.ac.uk.test
+
+import ebi.ac.uk.extended.model.ExtProcessingStatus.PROCESSED
+import ebi.ac.uk.extended.model.ExtSection
+import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.extended.model.ExtSubmissionMethod.PAGE_TAB
+import java.time.OffsetDateTime
+import java.time.ZoneOffset.UTC
+
+val basicExtSubmission = ExtSubmission(
+    accNo = "S-TEST123",
+    version = 1,
+    title = "Test Submission",
+    owner = "owner@email.org",
+    submitter = "submitter@email.org",
+    method = PAGE_TAB,
+    relPath = "S-TEST/123/S-TEST123",
+    rootPath = null,
+    released = false,
+    secretKey = "a-secret-key",
+    status = PROCESSED,
+    releaseTime = null,
+    modificationTime = OffsetDateTime.of(2018, 9, 21, 0, 0, 0, 0, UTC),
+    creationTime = OffsetDateTime.of(2018, 9, 21, 0, 0, 0, 0, UTC),
+    attributes = emptyList(),
+    tags = emptyList(),
+    collections = emptyList(),
+    section = ExtSection(type = "Study")
+)

submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/common/model/BasicCollection.kt

@@ -5,5 +5,6 @@ import java.time.OffsetDateTime
 data class BasicCollection(
     val accNo: String,
     val accNoPattern: String,
+    val validator: String?,
     val releaseTime: OffsetDateTime?
 )

submission/persistence-common-api/src/main/kotlin/ac/uk/ebi/biostd/persistence/exception/CollectionNotFoundException.kt

@@ -2,8 +2,12 @@ package ac.uk.ebi.biostd.persistence.exception
 
 class CollectionNotFoundException(
     collection: String
-) : RuntimeException("The project '$collection' was not found")
+) : RuntimeException("The collection '$collection' was not found")
 
 class CollectionWithoutPatternException(
     collection: String
-) : RuntimeException("The project '$collection' does not have a valid accession pattern")
+) : RuntimeException("The collection '$collection' does not have a valid accession pattern")
+
+class CollectionValidationException(
+    errors: List<String>
+) : RuntimeException("The submission doesn't comply with the collection requirements. Errors: $errors")

submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoMetaQueryService.kt

@@ -8,17 +8,20 @@ import ac.uk.ebi.biostd.persistence.doc.model.DocSubmission
 import ac.uk.ebi.biostd.persistence.doc.model.asBasicSubmission
 import ac.uk.ebi.biostd.persistence.exception.CollectionNotFoundException
 import ac.uk.ebi.biostd.persistence.exception.CollectionWithoutPatternException
+import ebi.ac.uk.model.constants.SubFields
 import ebi.ac.uk.model.constants.SubFields.ACC_NO_TEMPLATE
-import java.time.ZoneOffset
+import ebi.ac.uk.model.constants.SubFields.COLLECTION_VALIDATOR
+import java.time.ZoneOffset.UTC
 
 class SubmissionMongoMetaQueryService(
     private val submissionRepository: SubmissionDocDataRepository
 ) : SubmissionMetaQueryService {
-
     override fun getBasicCollection(accNo: String): BasicCollection {
         val collection = submissionRepository.findByAccNo(accNo) ?: throw CollectionNotFoundException(accNo)
-        val collectionPattern = collection.accTemplate() ?: throw CollectionWithoutPatternException(accNo)
-        return BasicCollection(collection.accNo, collectionPattern, collection.releaseTime?.atOffset(ZoneOffset.UTC))
+        val collectionPattern = collection.attrValue(ACC_NO_TEMPLATE) ?: throw CollectionWithoutPatternException(accNo)
+        val validator = collection.attrValue(COLLECTION_VALIDATOR)
+
+        return BasicCollection(collection.accNo, collectionPattern, validator, collection.releaseTime?.atOffset(UTC))
     }
 
     override fun findLatestBasicByAccNo(accNo: String): BasicSubmission? =
@@ -29,5 +32,5 @@ class SubmissionMongoMetaQueryService(
 
     override fun existByAccNo(accNo: String): Boolean = submissionRepository.existsByAccNo(accNo)
 
-    private fun DocSubmission.accTemplate() = attributes.firstOrNull { it.name == ACC_NO_TEMPLATE.value }?.value
+    private fun DocSubmission.attrValue(name: SubFields) = attributes.firstOrNull { it.name == name.value }?.value
 }

submission/persistence-mongo/src/test/kotlin/ac/uk/ebi/biostd/persistence/doc/service/SubmissionMongoMetaQueryServiceTest.kt

@@ -8,15 +8,16 @@ import ac.uk.ebi.biostd.persistence.doc.integration.MongoDbServicesConfig
 import ac.uk.ebi.biostd.persistence.doc.model.DocAttribute
 import ac.uk.ebi.biostd.persistence.doc.model.DocProcessingStatus.PROCESSED
 import ac.uk.ebi.biostd.persistence.doc.service.SubmissionMongoMetaQueryServiceTest.TestConfig
+import ac.uk.ebi.biostd.persistence.doc.test.doc.RELEASE_TIME
 import ac.uk.ebi.biostd.persistence.doc.test.doc.testDocSubmission
 import ac.uk.ebi.biostd.persistence.exception.CollectionNotFoundException
 import ac.uk.ebi.biostd.persistence.exception.CollectionWithoutPatternException
 import ebi.ac.uk.db.MONGO_VERSION
-import ebi.ac.uk.model.constants.SubFields
+import ebi.ac.uk.model.constants.SubFields.ACC_NO_TEMPLATE
+import ebi.ac.uk.model.constants.SubFields.COLLECTION_VALIDATOR
 import org.assertj.core.api.Assertions.assertThat
-import org.assertj.core.api.Assertions.assertThatExceptionOfType
-import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.extension.ExtendWith
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
@@ -32,6 +33,7 @@ import org.testcontainers.junit.jupiter.Testcontainers
 import org.testcontainers.utility.DockerImageName
 import uk.ac.ebi.extended.serialization.service.ExtSerializationService
 import java.nio.file.Files
+import java.time.ZoneOffset.UTC
 
 @ExtendWith(SpringExtension::class)
 @Testcontainers
@@ -40,38 +42,41 @@ internal class SubmissionMongoMetaQueryServiceTest(
     @Autowired val submissionMongoRepository: SubmissionMongoRepository,
     @Autowired val testInstance: SubmissionMongoMetaQueryService
 ) {
+    @Test
+    fun getBasicCollection() {
+        submissionMongoRepository.save(testDocSubmission.copy(
+            accNo = "EuToxRisk",
+            version = 1,
+            status = PROCESSED,
+            attributes = listOf(
+                DocAttribute(ACC_NO_TEMPLATE.value, "!{S-TOX}"),
+                DocAttribute(COLLECTION_VALIDATOR.value, "EuToxRiskValidator"))
+        ))
+
+        val (accNo, accNoPattern, validator, releaseTime) = testInstance.getBasicCollection("EuToxRisk")
+        assertThat(accNo).isEqualTo("EuToxRisk")
+        assertThat(accNoPattern).isEqualTo("!{S-TOX}")
+        assertThat(validator).isEqualTo("EuToxRiskValidator")
+        assertThat(releaseTime).isEqualTo(RELEASE_TIME.atOffset(UTC))
+    }
 
-    @Nested
-    inner class GetBasicCollection {
-        @Test
-        fun getBasicProject() {
-            submissionMongoRepository.save(testDocSubmission.copy(
-                accNo = "accNo1",
-                version = 1,
-                status = PROCESSED,
-                attributes = listOf(DocAttribute(SubFields.ACC_NO_TEMPLATE.value, "template"))
-            ))
-
-            val result = testInstance.getBasicCollection("accNo1")
-
-            assertThat(result.accNo).isEqualTo("accNo1")
-        }
-
-        @Test
-        fun `get basic project when exist without project pattern`() {
-            submissionMongoRepository.save(testDocSubmission.copy(accNo = "accNo1A", version = 1, status = PROCESSED))
-
-            assertThatExceptionOfType(CollectionWithoutPatternException::class.java)
-                .isThrownBy { testInstance.getBasicCollection("accNo1A") }
-                .withMessage("The project 'accNo1A' does not have a valid accession pattern")
-        }
+    @Test
+    fun `non existing collection`() {
+        val error = assertThrows<CollectionNotFoundException> { testInstance.getBasicCollection("NonExisting") }
+        assertThat(error.message).isEqualTo("The collection 'NonExisting' was not found")
+    }
 
-        @Test
-        fun `get basic project when does not exist`() {
-            assertThatExceptionOfType(CollectionNotFoundException::class.java)
-                .isThrownBy { testInstance.getBasicCollection("accNo1B") }
-                .withMessage("The project 'accNo1B' was not found")
-        }
+    @Test
+    fun `collection without pattern`() {
+        submissionMongoRepository.save(testDocSubmission.copy(
+            accNo = "PatternLess",
+            version = 1,
+            status = PROCESSED,
+            attributes = listOf(DocAttribute(COLLECTION_VALIDATOR.value, "PatternLessValidator"))
+        ))
+
+        val error = assertThrows<CollectionWithoutPatternException> { testInstance.getBasicCollection("PatternLess") }
+        assertThat(error.message).isEqualTo("The collection 'PatternLess' does not have a valid accession pattern")
     }
 
     @Test
@@ -103,7 +108,6 @@ internal class SubmissionMongoMetaQueryServiceTest(
     @Configuration
     @Import(MongoDbServicesConfig::class)
     class TestConfig {
-
         @Bean
         fun applicationProperties(): ApplicationProperties {
             val properties = ApplicationProperties()

submission/persistence-sql/src/main/kotlin/ac/uk/ebi/biostd/persistence/integration/services/SubmissionSqlQueryService.kt

@@ -5,25 +5,27 @@ import ac.uk.ebi.biostd.persistence.common.model.BasicSubmission
 import ac.uk.ebi.biostd.persistence.common.service.SubmissionMetaQueryService
 import ac.uk.ebi.biostd.persistence.exception.CollectionNotFoundException
 import ac.uk.ebi.biostd.persistence.exception.CollectionWithoutPatternException
+import ac.uk.ebi.biostd.persistence.model.DbSubmission
 import ac.uk.ebi.biostd.persistence.repositories.AccessTagDataRepo
 import ac.uk.ebi.biostd.persistence.repositories.SubmissionDataRepository
 import ac.uk.ebi.biostd.persistence.repositories.data.CollectionSqlDataService.Companion.asBasicSubmission
 import ebi.ac.uk.model.constants.SubFields
+import ebi.ac.uk.model.constants.SubFields.ACC_NO_TEMPLATE
+import ebi.ac.uk.model.constants.SubFields.COLLECTION_VALIDATOR
 
 @Suppress("TooManyFunctions")
 internal class SubmissionSqlQueryService(
     private val subRepository: SubmissionDataRepository,
     private val accessTagDataRepo: AccessTagDataRepo
 ) : SubmissionMetaQueryService {
     override fun getBasicCollection(accNo: String): BasicCollection {
-        val collectionDb = subRepository.findBasicWithAttributes(accNo)
-        require(collectionDb != null) { throw CollectionNotFoundException(accNo) }
+        val collection = subRepository.findBasicWithAttributes(accNo)
+        require(collection != null) { throw CollectionNotFoundException(accNo) }
 
-        val collectionPattern =
-            collectionDb.attributes.firstOrNull { it.name == SubFields.ACC_NO_TEMPLATE.value }
-                ?.value ?: throw CollectionWithoutPatternException(accNo)
+        val validator = collection.attrValue(COLLECTION_VALIDATOR)
+        val collectionPattern = collection.attrValue(ACC_NO_TEMPLATE) ?: throw CollectionWithoutPatternException(accNo)
 
-        return BasicCollection(collectionDb.accNo, collectionPattern, collectionDb.releaseTime)
+        return BasicCollection(collection.accNo, collectionPattern, validator, collection.releaseTime)
     }
 
     override fun findLatestBasicByAccNo(accNo: String): BasicSubmission? =
@@ -32,4 +34,6 @@ internal class SubmissionSqlQueryService(
     override fun getAccessTags(accNo: String) = accessTagDataRepo.findBySubmissionsAccNo(accNo).map { it.name }
 
     override fun existByAccNo(accNo: String): Boolean = subRepository.existsByAccNo(accNo)
+
+    private fun DbSubmission.attrValue(name: SubFields) = attributes.firstOrNull { it.name == name.value }?.value
 }

submission/submission-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/ApplicationProperties.kt

@@ -21,6 +21,9 @@ open class ApplicationProperties {
 
     @NestedConfigurationProperty
     var mongo: MongoModuleProperties = MongoModuleProperties()
+
+    @NestedConfigurationProperty
+    var validator: ValidatorProperties = ValidatorProperties()
 }
 
 class FireProperties {
@@ -32,3 +35,7 @@ class FireProperties {
 class MongoModuleProperties {
     lateinit var connection: String
 }
+
+class ValidatorProperties {
+    lateinit var euToxRiskValidationApi: String
+}