Pivotal ID # 184977702: Assign DOIs Automatically

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/constants/Fields.kt

@@ -46,6 +46,7 @@ enum class SubFields(override val value: String) : Fields {
     TITLE("Title"),
     ROOT_PATH("RootPath"),
     PUBLIC_ACCESS_TAG("Public"),
+    DOI_REQUESTED("RequestDOI"),
 
     RELEASE_DATE("ReleaseDate"),
     RELEASE_TIME("rtime"),

submission/submission-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/ApplicationProperties.kt

@@ -20,6 +20,7 @@ data class ApplicationProperties(
     val validator: ValidatorProperties,
     val persistence: PersistenceProperties,
     val notifications: NotificationsProperties,
+    val doi: DoiProperties,
 )
 
 data class RetryProperties(
@@ -58,3 +59,10 @@ data class NotificationsProperties(
     val requestQueue: String,
     val requestRoutingKey: String,
 )
+
+data class DoiProperties(
+    val endpoint: String,
+    val uiUrl: String,
+    val user: String,
+    val password: String
+)

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/itest/ITestListener.kt

@@ -36,6 +36,7 @@ class ITestListener : TestExecutionListener {
         fireSetup()
         ftpSetup()
         appPropertiesSetup()
+        doiSetup()
     }
 
     override fun testPlanExecutionFinished(testPlan: TestPlan) {
@@ -97,6 +98,13 @@ class ITestListener : TestExecutionListener {
         System.setProperty("app.persistence.enableFire", "${System.getProperty("enableFire").toBoolean()}")
     }
 
+    private fun doiSetup() {
+        System.setProperty("app.doi.endpoint", "https://test.crossref.org/servlet/deposit")
+        System.setProperty("app.doi.uiUrl", "https://www.ebi.ac.uk/biostudies/")
+        System.setProperty("app.doi.user", "a-user")
+        System.setProperty("app.doi.password", "a-password")
+    }
+
     companion object {
         private val testAppFolder = Files.createTempDirectory("test-app-folder").toFile()
         private const val defaultBucket = "bio-fire-bucket"

submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmitterConfig.kt

@@ -16,6 +16,7 @@ import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService
 import ac.uk.ebi.biostd.persistence.filesystem.pagetab.PageTabService
 import ac.uk.ebi.biostd.submission.service.AccNoService
 import ac.uk.ebi.biostd.submission.service.CollectionProcessor
+import ac.uk.ebi.biostd.submission.service.DoiService
 import ac.uk.ebi.biostd.submission.service.FileSourcesService
 import ac.uk.ebi.biostd.submission.service.TimesService
 import ac.uk.ebi.biostd.submission.submitter.ExtSubmissionSubmitter
@@ -172,11 +173,13 @@ class SubmitterConfig(
 
     @Bean
     fun submissionSubmitter(
+        doiService: DoiService,
         extSubmissionSubmitter: ExtSubmissionSubmitter,
         submissionProcessor: SubmissionProcessor,
         collectionValidationService: CollectionValidationService,
         draftService: SubmissionDraftPersistenceService,
     ): SubmissionSubmitter = SubmissionSubmitter(
+        doiService,
         extSubmissionSubmitter,
         submissionProcessor,
         collectionValidationService,
@@ -227,7 +230,6 @@ class SubmitterConfig(
     }
 
     @Configuration
-    @Suppress("MagicNumber")
     @EnableConfigurationProperties
     class ServiceConfig(
         private val service: PersistenceService,
@@ -238,6 +240,9 @@ class SubmitterConfig(
         @Bean
         fun accNoPatternUtil() = AccNoPatternUtil()
 
+        @Bean
+        fun doiService(webClient: WebClient) = DoiService(webClient, properties.doi)
+
         @Bean
         fun accNoService() = AccNoService(service, accNoPatternUtil(), userPrivilegesService, properties.subBasePath)
 

submission/submission-webapp/src/main/resources/application-local.yml

@@ -54,3 +54,8 @@ app:
     magicDirPath: # Absolute path to the folder to be used for the user/groups magic folders links
     environment: LOCAL
     requireActivation: false
+  doi:
+    endpoint: https://test-endpoint.org
+    uiUrl: https://www.biostudies.ac.uk
+    user: a-user
+    password: a-password

submission/submission-webapp/src/main/resources/application.yml

@@ -110,3 +110,8 @@ app:
   notifications:
     requestQueue: submission-request-submitter-queue # queue used to handle request stages
     requestRoutingKey: bio.submission.requested # request messages routing key
+  doi:
+    endpoint: # DOI registration endpoint
+    uiUrl: # The BioStudies UI URL which will be associated to the DOI record
+    user: # User for the DOI service
+    password: # Password for the DOI service

submission/submitter/build.gradle.kts

@@ -19,6 +19,7 @@ import SpringBootDependencies.SpringBootStarterDataJpa
 import SpringBootDependencies.SpringBootStarterWeb
 import TestDependencies.BaseTestCompileDependencies
 import TestDependencies.BaseTestRuntimeDependencies
+import TestDependencies.KotlinXmlBuilder
 import io.spring.gradle.dependencymanagement.dsl.DependencyManagementExtension
 import org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES
 
@@ -50,6 +51,7 @@ dependencies {
     implementation(KotlinReflect)
     implementation(KotlinStdLib)
     implementation(KotlinLogging)
+    implementation(KotlinXmlBuilder)
 
     implementation(SpringBootStarterDataJpa)
     implementation(SpringBootStarterWeb)

submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/exceptions/DoiExceptions.kt

@@ -0,0 +1,20 @@
+package ac.uk.ebi.biostd.submission.exceptions
+
+class MissingDoiFieldException(field: String) : RuntimeException("The required DOI field '$field' could not be found")
+
+class MissingTitleException : RuntimeException("A title is required for DOI registration")
+
+class InvalidOrgNamesException(
+    organizations: List<String>
+) : RuntimeException("The following organization names are empty: ${organizations.joinToString(", ")}")
+
+class InvalidOrgException : RuntimeException("Organizations are required to have an accession")
+
+class InvalidAuthorNameException : RuntimeException("Authors are required to have a name")
+
+class MissingAuthorAffiliationException : RuntimeException("Authors are required to have an affiliation")
+
+class InvalidAuthorAffiliationException(
+    author: String,
+    organization: String,
+) : RuntimeException("The organization '$organization' affiliated to the author '$author' could not be found")

submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/model/DoiRequest.kt

@@ -0,0 +1,80 @@
+package ac.uk.ebi.biostd.submission.model
+
+import ebi.ac.uk.util.collections.ifNotEmpty
+import org.redundent.kotlin.xml.xml
+import java.time.Instant
+
+internal data class Contributor(
+    val name: String,
+    val surname: String,
+    val affiliation: String,
+    val orcid: String?
+)
+
+internal class DoiRequest(
+    private val accNo: String,
+    private val title: String,
+    private val instanceUrl: String,
+    private val contributors: List<Contributor>,
+) {
+    fun asXmlRequest(): String {
+        val timestamp = Instant.now().epochSecond.toString()
+        return xml("doi_batch") {
+            xmlns = "http://www.crossref.org/schema/4.4.1"
+            "head" {
+                "doi_batch_id" { -timestamp }
+                "timestamp" { -timestamp }
+                "depositor" {
+                    "depositor_name" { -DEPOSITOR }
+                    "email_address" { -EMAIL }
+                }
+                "registrant" { -DEPOSITOR }
+            }
+            "body" {
+                "database" {
+                    "database_metadata" {
+                        attribute("language", "en")
+                        "titles" {
+                            "title" { -BS_TITLE }
+                        }
+                    }
+                    "dataset" {
+                        contributors.ifNotEmpty {
+                            "contributors" {
+                                contributors.forEachIndexed { index, contributor ->
+                                    "person_name" {
+                                        attribute("contributor_role", "author")
+                                        attribute("sequence", index)
+                                        "given_name" { -contributor.name }
+                                        "surname" { -contributor.surname }
+                                        "affiliation" { -contributor.affiliation }
+                                        contributor.orcid?.let { orcid ->
+                                            "ORCID" {
+                                                attribute("authenticated", "false")
+                                                -orcid
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        "titles" {
+                            "title" { -title }
+                        }
+                        "doi_data" {
+                            "doi" { -"$BS_DOI_ID/$accNo" }
+                            "resource" { -"$instanceUrl/studies/$accNo" }
+                        }
+                    }
+                }
+            }
+        }.toString()
+    }
+
+    companion object {
+        const val BS_DOI_ID = "10.6019"
+        const val BS_TITLE = "BioStudies Database"
+        const val DEPOSITOR = "EMBL-EBI"
+        const val EMAIL = "biostudies@ebi.ac.uk"
+    }
+}

submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/service/DoiService.kt

@@ -0,0 +1,102 @@
+package ac.uk.ebi.biostd.submission.service
+
+import ac.uk.ebi.biostd.common.properties.DoiProperties
+import ac.uk.ebi.biostd.submission.exceptions.InvalidAuthorAffiliationException
+import ac.uk.ebi.biostd.submission.exceptions.InvalidAuthorNameException
+import ac.uk.ebi.biostd.submission.exceptions.InvalidOrgException
+import ac.uk.ebi.biostd.submission.exceptions.InvalidOrgNamesException
+import ac.uk.ebi.biostd.submission.exceptions.MissingAuthorAffiliationException
+import ac.uk.ebi.biostd.submission.exceptions.MissingDoiFieldException
+import ac.uk.ebi.biostd.submission.exceptions.MissingTitleException
+import ac.uk.ebi.biostd.submission.model.Contributor
+import ac.uk.ebi.biostd.submission.model.DoiRequest
+import ebi.ac.uk.commons.http.ext.RequestParams
+import ebi.ac.uk.commons.http.ext.post
+import ebi.ac.uk.extended.model.ExtSection
+import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.extended.model.allSections
+import ebi.ac.uk.extended.model.computedTitle
+import ebi.ac.uk.io.FileUtils
+import ebi.ac.uk.util.collections.ifNotEmpty
+import mu.KotlinLogging
+import org.springframework.core.io.FileSystemResource
+import org.springframework.util.LinkedMultiValueMap
+import org.springframework.web.reactive.function.client.WebClient
+import java.nio.file.Files
+
+internal const val AFFILIATION_ATTR = "affiliation"
+internal const val NAME_ATTR = "name"
+internal const val ORCID_ATTR = "orcid"
+
+internal const val ORG_TYPE = "organization"
+internal const val AUTHOR_TYPE = "author"
+
+internal const val FILE_PARAM = "fname"
+internal const val OPERATION_PARAM = "operation"
+internal const val OPERATION_PARAM_VALUE = "doMDUpload"
+internal const val PASSWORD_PARAM = "login_password"
+internal const val USER_PARAM = "login_id"
+internal const val TEMP_FILE_NAME = "doi-request"
+
+private val logger = KotlinLogging.logger {}
+
+@Suppress("ThrowsCount")
+class DoiService(
+    private val webClient: WebClient,
+    private val properties: DoiProperties,
+) {
+    fun registerDoi(sub: ExtSubmission) {
+        val title = requireNotNull(sub.computedTitle) { throw MissingTitleException() }
+        val request = DoiRequest(sub.accNo, title, properties.uiUrl, getContributors(sub))
+        val requestFile = Files.createTempFile("${TEMP_FILE_NAME}_${sub.accNo}", ".xml").toFile()
+        FileUtils.writeContent(requestFile, request.asXmlRequest())
+
+        val body = LinkedMultiValueMap<String, Any>().apply {
+            add(USER_PARAM, properties.user)
+            add(PASSWORD_PARAM, properties.password)
+            add(OPERATION_PARAM, OPERATION_PARAM_VALUE)
+            add(FILE_PARAM, FileSystemResource(requestFile))
+        }
+
+        logger.info { "${sub.accNo} ${sub.owner} Registering DOI" }
+        webClient.post(properties.endpoint, RequestParams(body = body))
+    }
+
+    private fun getContributors(sub: ExtSubmission): List<Contributor> {
+        val organizations = getOrganizations(sub)
+        return sub.allSections
+            .filter { it.type.lowercase() == AUTHOR_TYPE }
+            .ifEmpty { throw MissingDoiFieldException(AUTHOR_TYPE) }
+            .map { it.asContributor(organizations) }
+    }
+
+    private fun ExtSection.asContributor(orgs: Map<String, String>): Contributor {
+        val attrsMap = attributes.associateBy({ it.name.lowercase() }, { it.value })
+        val names = requireNotNull(attrsMap[NAME_ATTR]) { throw InvalidAuthorNameException() }
+        val affiliation = requireNotNull(attrsMap[AFFILIATION_ATTR]) { throw MissingAuthorAffiliationException() }
+        val org = requireNotNull(orgs[affiliation]) { throw InvalidAuthorAffiliationException(names, affiliation) }
+
+        return Contributor(
+            name = names.substringBeforeLast(" ", ""),
+            surname = names.substringAfterLast(" "),
+            affiliation = org,
+            orcid = attrsMap[ORCID_ATTR]
+        )
+    }
+
+    private fun getOrganizations(sub: ExtSubmission): Map<String, String> {
+        val organizations = sub.allSections
+            .filter { it.type.lowercase() == ORG_TYPE }
+            .ifEmpty { throw MissingDoiFieldException(ORG_TYPE) }
+            .associateBy(
+                { requireNotNull(it.accNo) { throw InvalidOrgException() } },
+                { org -> org.attributes.find { it.name.lowercase() == NAME_ATTR }?.value.orEmpty() },
+            )
+
+        organizations.entries
+            .filter { it.value.isEmpty() }
+            .ifNotEmpty { entries -> throw InvalidOrgNamesException(entries.map { it.key }) }
+
+        return organizations
+    }
+}

submission/submitter/src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/SubmissionSubmitter.kt

@@ -4,6 +4,7 @@ import ac.uk.ebi.biostd.persistence.common.request.ExtSubmitRequest
 import ac.uk.ebi.biostd.persistence.common.service.SubmissionDraftPersistenceService
 import ac.uk.ebi.biostd.submission.exceptions.InvalidSubmissionException
 import ac.uk.ebi.biostd.submission.model.SubmitRequest
+import ac.uk.ebi.biostd.submission.service.DoiService
 import ac.uk.ebi.biostd.submission.validator.collection.CollectionValidationService
 import ebi.ac.uk.extended.events.RequestCheckedReleased
 import ebi.ac.uk.extended.events.RequestCleaned
@@ -13,12 +14,14 @@ import ebi.ac.uk.extended.events.RequestIndexed
 import ebi.ac.uk.extended.events.RequestLoaded
 import ebi.ac.uk.extended.events.RequestPersisted
 import ebi.ac.uk.extended.model.ExtSubmission
+import ebi.ac.uk.model.constants.SubFields.DOI_REQUESTED
 import mu.KotlinLogging
 
 private val logger = KotlinLogging.logger {}
 
 @Suppress("TooManyFunctions")
 class SubmissionSubmitter(
+    private val doiService: DoiService,
     private val submissionSubmitter: ExtSubmissionSubmitter,
     private val submissionProcessor: SubmissionProcessor,
     private val collectionValidationService: CollectionValidationService,
@@ -74,6 +77,7 @@ class SubmissionSubmitter(
             rqt.draftKey?.let { startProcessingDraft(rqt.accNo, rqt.owner, it) }
             val processed = submissionProcessor.processSubmission(rqt)
             collectionValidationService.executeCollectionValidators(processed)
+            if (processed.requiresDoi) doiService.registerDoi(processed)
             rqt.draftKey?.let { acceptDraft(rqt.accNo, rqt.owner, it) }
             logger.info { "${rqt.accNo} ${rqt.owner} Finished processing submission request" }
 
@@ -99,4 +103,7 @@ class SubmissionSubmitter(
         draftService.setActiveStatus(draftKey)
         logger.info { "$accNo $owner Status of draft with key '$draftKey' set to ACTIVE" }
     }
+
+    private val ExtSubmission.requiresDoi: Boolean
+        get() = attributes.find { it.name == DOI_REQUESTED.value } != null
 }