From 9ed9aaced23a6677ee98836f99f0aeb440fb6679 Mon Sep 17 00:00:00 2001 From: Jhoan Munoz Date: Sat, 25 Nov 2023 02:07:57 +0000 Subject: [PATCH 1/3] Pivotal ID # 186425798: Regenerate all PMC submission FTP - Make suspend the endpoints and repositories for the releaser service - Change the generateFtpLinks client operation to receive the accNo instead of the relPath --- .../client/api/CommonOperationsClient.kt | 9 +-- .../client/integration/web/SubmitClient.kt | 2 +- .../uk/commons/http/ext/WebClientExtAsync.kt | 9 +++ .../submission-releaser-task/build.gradle.kts | 2 + .../releaser/SubmissionReleaserApp.kt | 11 ++-- .../releaser/config/ApplicationConfig.kt | 7 ++- .../scheduler/releaser/model/ReleaseData.kt | 11 ---- .../service/SubmissionReleaserService.kt | 55 ++++++++++++------- .../service/SubmissionReleaserServiceTest.kt | 26 +++++---- .../SubmissionReleaserRepository.kt | 15 ++--- .../doc/db/repositories/SingleFieldDtos.kt | 6 ++ .../resources/SubmissionOperationsResource.kt | 5 +- 12 files changed, 96 insertions(+), 62 deletions(-) delete mode 100644 scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/model/ReleaseData.kt rename scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/persistence/ReleaserRepository.kt => submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/SubmissionReleaserRepository.kt (56%) diff --git a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt index 05f8e789d7..bc3d4fc8e8 100644 --- a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt +++ b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt @@ -2,13 +2,14 @@ package ac.uk.ebi.biostd.client.api import ac.uk.ebi.biostd.client.integration.web.GeneralOperations import ebi.ac.uk.api.dto.UserGroupDto +import ebi.ac.uk.commons.http.builder.linkedMultiValueMapOf import ebi.ac.uk.commons.http.ext.RequestParams import ebi.ac.uk.commons.http.ext.getForObject +import ebi.ac.uk.commons.http.ext.postAsync import ebi.ac.uk.commons.http.ext.postForObject import ebi.ac.uk.commons.http.ext.put import ebi.ac.uk.model.Collection import ebi.ac.uk.model.Group -import org.springframework.util.LinkedMultiValueMap import org.springframework.web.reactive.function.client.WebClient private const val GROUP_URL = "/groups" @@ -26,9 +27,9 @@ class CommonOperationsClient( return client.getForObject>(PROJECTS_URL).toList() } - override fun generateFtpLink(relPath: String) { - val body = LinkedMultiValueMap(mapOf("relPath" to listOf(relPath))) - client.postForObject("$FTP_URL/generate", RequestParams(body = body)) + override suspend fun generateFtpLinks(accNo: String) { + val body = linkedMultiValueMapOf("accNo" to listOf(accNo)) + client.postAsync("$FTP_URL/generate", RequestParams(body = body)) } override fun createGroup(groupName: String, groupDescription: String): UserGroupDto { diff --git a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/integration/web/SubmitClient.kt b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/integration/web/SubmitClient.kt index c4242d2888..a572f90f07 100644 --- a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/integration/web/SubmitClient.kt +++ b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/integration/web/SubmitClient.kt @@ -83,7 +83,7 @@ interface SecurityOperations { interface GeneralOperations { fun getGroups(): List fun getCollections(): List - fun generateFtpLink(relPath: String) + suspend fun generateFtpLinks(accNo: String) fun createGroup(groupName: String, groupDescription: String): UserGroupDto fun addUserInGroup(groupName: String, userName: String) } diff --git a/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/ext/WebClientExtAsync.kt b/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/ext/WebClientExtAsync.kt index 9b064714ab..c1bf4e19aa 100644 --- a/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/ext/WebClientExtAsync.kt +++ b/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/ext/WebClientExtAsync.kt @@ -18,6 +18,15 @@ suspend inline fun WebClient.putForObjectAsync(url: String, params: return put().retrieveAsync(url, params)!! } +suspend fun WebClient.postAsync(url: String, params: RequestParams? = null) { + val uriSpec = post().uri(url) + params?.headers?.let { headers -> uriSpec.headers { it.addAll(headers) } } + params?.body?.let { body -> uriSpec.bodyValue(body) } + + val response = uriSpec.awaitExchange() + require(response.statusCode().isError.not()) { response.toString() } +} + suspend fun WebClient.deleteAsync(url: String) { val response = delete().uri(url).awaitExchange() require(response.statusCode().isError.not()) { response.toString() } diff --git a/scheduler/tasks/submission-releaser-task/build.gradle.kts b/scheduler/tasks/submission-releaser-task/build.gradle.kts index 9df986e1c0..0389804501 100644 --- a/scheduler/tasks/submission-releaser-task/build.gradle.kts +++ b/scheduler/tasks/submission-releaser-task/build.gradle.kts @@ -1,3 +1,4 @@ +import Dependencies.KotlinCoroutines import Dependencies.KotlinLogging import Dependencies.KotlinStdLib import Projects.ClientBioWebClient @@ -29,6 +30,7 @@ dependencies { api(project(SubmissionPersistenceCommonApi)) api(project(SubmissionPersistenceMongo)) + implementation(KotlinCoroutines) implementation(KotlinLogging) implementation(KotlinStdLib) implementation(SpringBootStarterAmqp) diff --git a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/SubmissionReleaserApp.kt b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/SubmissionReleaserApp.kt index 81abf2ae5c..280f27cf9e 100644 --- a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/SubmissionReleaserApp.kt +++ b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/SubmissionReleaserApp.kt @@ -3,6 +3,7 @@ package uk.ac.ebi.scheduler.releaser import ac.uk.ebi.scheduler.properties.ReleaserMode.GENERATE_FTP_LINKS import ac.uk.ebi.scheduler.properties.ReleaserMode.NOTIFY import ac.uk.ebi.scheduler.properties.ReleaserMode.RELEASE +import kotlinx.coroutines.runBlocking import org.springframework.boot.CommandLineRunner import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -30,10 +31,12 @@ class SubmissionReleaserExecutor( private lateinit var context: ApplicationContext override fun run(vararg args: String?) { - when (applicationProperties.mode) { - NOTIFY -> submissionReleaserService.notifySubmissionReleases() - RELEASE -> submissionReleaserService.releaseDailySubmissions() - GENERATE_FTP_LINKS -> submissionReleaserService.generateFtpLinks() + runBlocking { + when (applicationProperties.mode) { + NOTIFY -> submissionReleaserService.notifySubmissionReleases() + RELEASE -> submissionReleaserService.releaseDailySubmissions() + GENERATE_FTP_LINKS -> submissionReleaserService.generateFtpLinks() + } } } diff --git a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/config/ApplicationConfig.kt b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/config/ApplicationConfig.kt index 19d07f08e2..414a546cf9 100644 --- a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/config/ApplicationConfig.kt +++ b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/config/ApplicationConfig.kt @@ -2,18 +2,21 @@ package uk.ac.ebi.scheduler.releaser.config import ac.uk.ebi.biostd.client.integration.web.BioWebClient import ac.uk.ebi.biostd.client.integration.web.SecurityWebClient +import ac.uk.ebi.biostd.persistence.doc.MongoDbReactiveConfig +import ac.uk.ebi.biostd.persistence.doc.db.reactive.repositories.SubmissionReleaserRepository import org.springframework.amqp.rabbit.core.RabbitTemplate import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Import import uk.ac.ebi.events.config.EventsProperties import uk.ac.ebi.scheduler.releaser.SubmissionReleaserExecutor -import uk.ac.ebi.scheduler.releaser.persistence.ReleaserRepository import uk.ac.ebi.scheduler.releaser.service.EventsPublisherService import uk.ac.ebi.scheduler.releaser.service.SubmissionReleaserService @Configuration +@Import(MongoDbReactiveConfig::class) class ApplicationConfig( - private val releaserRepository: ReleaserRepository, + private val releaserRepository: SubmissionReleaserRepository, private val appProperties: ApplicationProperties, ) { @Bean diff --git a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/model/ReleaseData.kt b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/model/ReleaseData.kt deleted file mode 100644 index ac1fb8b0bf..0000000000 --- a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/model/ReleaseData.kt +++ /dev/null @@ -1,11 +0,0 @@ -package uk.ac.ebi.scheduler.releaser.model - -import ac.uk.ebi.biostd.client.dto.ReleaseRequestDto - -data class ReleaseData( - val accNo: String, - val owner: String, - val relPath: String -) { - fun asReleaseDto() = ReleaseRequestDto(accNo, owner, relPath) -} diff --git a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserService.kt b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserService.kt index bc4a2a6413..b7fb54a9ef 100644 --- a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserService.kt +++ b/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserService.kt @@ -1,13 +1,18 @@ package uk.ac.ebi.scheduler.releaser.service +import ac.uk.ebi.biostd.client.dto.ReleaseRequestDto import ac.uk.ebi.biostd.client.integration.web.BioWebClient +import ac.uk.ebi.biostd.persistence.doc.db.reactive.repositories.SubmissionReleaserRepository +import ac.uk.ebi.biostd.persistence.doc.db.repositories.ReleaseData import ebi.ac.uk.util.date.asOffsetAtEndOfDay import ebi.ac.uk.util.date.asOffsetAtStartOfDay import ebi.ac.uk.util.date.toDate +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.withContext import mu.KotlinLogging import uk.ac.ebi.scheduler.releaser.config.NotificationTimes -import uk.ac.ebi.scheduler.releaser.model.ReleaseData -import uk.ac.ebi.scheduler.releaser.persistence.ReleaserRepository import java.time.Instant import java.time.temporal.ChronoUnit.DAYS @@ -16,29 +21,35 @@ private val logger = KotlinLogging.logger {} class SubmissionReleaserService( private val bioWebClient: BioWebClient, private val notificationTimes: NotificationTimes, - private val releaserRepository: ReleaserRepository, + private val releaserRepository: SubmissionReleaserRepository, private val eventsPublisherService: EventsPublisherService, ) { - fun notifySubmissionReleases() { + suspend fun notifySubmissionReleases() { val today = Instant.now() notifyRelease(today.plus(notificationTimes.firstWarningDays, DAYS)) notifyRelease(today.plus(notificationTimes.secondWarningDays, DAYS)) notifyRelease(today.plus(notificationTimes.thirdWarningDays, DAYS)) } - fun releaseDailySubmissions() { + suspend fun releaseDailySubmissions() { val to = Instant.now().asOffsetAtEndOfDay() logger.info { "Releasing submissions up to $to" } - releaserRepository - .findAllUntil(to.toDate()) - .forEach(::releaseSafely) + withContext(Dispatchers.Default) { + releaserRepository + .findAllUntil(to.toDate()) + .map { async { releaseSafely(it) } } + .collect { it.await() } + } } - fun generateFtpLinks() { - releaserRepository - .findAllReleased() - .forEach(::generateFtpLink) + suspend fun generateFtpLinks() { + withContext(Dispatchers.Default) { + releaserRepository + .findAllReleased() + .map { async { generateFtpLinks(it) } } + .collect { it.await() } + } } private fun releaseSafely(releaseData: ReleaseData) { @@ -49,14 +60,20 @@ class SubmissionReleaserService( .onSuccess { logger.info { "Released submission ${releaseData.accNo}" } } } - private fun notifyRelease(date: Instant) { + private fun ReleaseData.asReleaseDto() = ReleaseRequestDto(accNo, owner, relPath) + + private suspend fun notifyRelease(date: Instant) { val from = date.asOffsetAtStartOfDay() val to = date.asOffsetAtEndOfDay() logger.info { "Notifying submissions releases from $from to $to" } - releaserRepository - .findAllBetween(from.toDate(), to.toDate()) - .forEach(::notify) + + withContext(Dispatchers.Default) { + releaserRepository + .findAllBetween(from.toDate(), to.toDate()) + .map { async { notify(it) } } + .collect { it.await() } + } } private fun notify(releaseData: ReleaseData) { @@ -64,8 +81,8 @@ class SubmissionReleaserService( eventsPublisherService.subToBePublished(releaseData.accNo, releaseData.owner) } - private fun generateFtpLink(releaseData: ReleaseData) { - logger.info { "Generating FTP link for submission ${releaseData.accNo}" } - bioWebClient.generateFtpLink(releaseData.relPath) + private suspend fun generateFtpLinks(releaseData: ReleaseData) { + logger.info { "Generating FTP links for submission ${releaseData.accNo}" } + bioWebClient.generateFtpLinks(releaseData.accNo) } } diff --git a/scheduler/tasks/submission-releaser-task/src/test/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserServiceTest.kt b/scheduler/tasks/submission-releaser-task/src/test/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserServiceTest.kt index 0099136f64..aa3aaa0edf 100644 --- a/scheduler/tasks/submission-releaser-task/src/test/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserServiceTest.kt +++ b/scheduler/tasks/submission-releaser-task/src/test/kotlin/uk/ac/ebi/scheduler/releaser/service/SubmissionReleaserServiceTest.kt @@ -2,23 +2,27 @@ package uk.ac.ebi.scheduler.releaser.service import ac.uk.ebi.biostd.client.dto.ReleaseRequestDto import ac.uk.ebi.biostd.client.integration.web.BioWebClient +import ac.uk.ebi.biostd.persistence.doc.db.reactive.repositories.SubmissionReleaserRepository +import ac.uk.ebi.biostd.persistence.doc.db.repositories.ReleaseData import ebi.ac.uk.util.date.asOffsetAtEndOfDay import ebi.ac.uk.util.date.toDate import io.mockk.clearAllMocks +import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.impl.annotations.MockK import io.mockk.junit5.MockKExtension import io.mockk.mockkStatic import io.mockk.slot import io.mockk.verify +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import uk.ac.ebi.scheduler.releaser.config.NotificationTimes -import uk.ac.ebi.scheduler.releaser.model.ReleaseData -import uk.ac.ebi.scheduler.releaser.persistence.ReleaserRepository import java.time.Instant import java.time.OffsetDateTime import java.time.ZoneOffset.UTC @@ -28,7 +32,7 @@ import java.util.Date class SubmissionReleaserServiceTest( @MockK private val bioWebClient: BioWebClient, @MockK private val notificationTimes: NotificationTimes, - @MockK private val releaserRepository: ReleaserRepository, + @MockK private val releaserRepository: SubmissionReleaserRepository, @MockK private val eventsPublisherService: EventsPublisherService, ) { private val mockNow = OffsetDateTime.of(2020, 9, 21, 10, 11, 0, 0, UTC).toInstant() @@ -46,7 +50,7 @@ class SubmissionReleaserServiceTest( } @Test - fun `notify submission release`() { + fun `notify submission release`() = runTest { val firstWarningData = ReleaseData("S-BSST0", "owner0@mail.org", "S-BSST/000/S-BSST0") val secondWarningData = ReleaseData("S-BSST1", "owner1@mail.org", "S-BSST/001/S-BSST1") val thirdWarningData = ReleaseData("S-BSST2", "owner2@mail.org", "S-BSST/002/S-BSST2") @@ -65,13 +69,13 @@ class SubmissionReleaserServiceTest( @Test fun `release daily submissions`( @MockK to: Date - ) { + ) = runTest { val requestSlot = slot() val released = ReleaseData("S-BSST0", "owner0@mail.org", "S-BSST/000/S-BSST0") every { mockNow.asOffsetAtEndOfDay().toDate() } returns to every { bioWebClient.releaseSubmission(capture(requestSlot)) } answers { nothing } - every { releaserRepository.findAllUntil(to) } returns listOf(released) + every { releaserRepository.findAllUntil(to) } returns flowOf(released) testInstance.releaseDailySubmissions() @@ -83,21 +87,21 @@ class SubmissionReleaserServiceTest( } @Test - fun `generate ftp links`() { + fun `generate ftp links`() = runTest { val released = ReleaseData("S-BSST0", "owner0@mail.org", "S-BSST/000/S-BSST0") - every { releaserRepository.findAllReleased() } returns listOf(released) - every { bioWebClient.generateFtpLink("S-BSST/000/S-BSST0") } answers { nothing } + every { releaserRepository.findAllReleased() } returns flowOf(released) + coEvery { bioWebClient.generateFtpLinks("S-BSST0") } answers { nothing } testInstance.generateFtpLinks() - verify(exactly = 1) { bioWebClient.generateFtpLink("S-BSST/000/S-BSST0") } + coVerify(exactly = 1) { bioWebClient.generateFtpLinks("S-BSST0") } } private fun mockNotificationQuery(month: Int, day: Int, response: ReleaseData) { val from = OffsetDateTime.of(2020, month, day, 0, 0, 0, 0, UTC).toDate() val to = OffsetDateTime.of(2020, month, day, 23, 59, 59, 0, UTC).toDate() - every { releaserRepository.findAllBetween(from, to) } returns listOf(response) + every { releaserRepository.findAllBetween(from, to) } returns flowOf(response) every { eventsPublisherService.subToBePublished(response.accNo, response.owner) } answers { nothing } } diff --git a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/persistence/ReleaserRepository.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/SubmissionReleaserRepository.kt similarity index 56% rename from scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/persistence/ReleaserRepository.kt rename to submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/SubmissionReleaserRepository.kt index 2d33f7406c..14973bcc61 100644 --- a/scheduler/tasks/submission-releaser-task/src/main/kotlin/uk/ac/ebi/scheduler/releaser/persistence/ReleaserRepository.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/reactive/repositories/SubmissionReleaserRepository.kt @@ -1,28 +1,29 @@ -package uk.ac.ebi.scheduler.releaser.persistence +package ac.uk.ebi.biostd.persistence.doc.db.reactive.repositories +import ac.uk.ebi.biostd.persistence.doc.db.repositories.ReleaseData import ac.uk.ebi.biostd.persistence.doc.model.DocSubmission +import kotlinx.coroutines.flow.Flow import org.bson.types.ObjectId import org.springframework.data.mongodb.repository.Query -import org.springframework.data.repository.PagingAndSortingRepository -import uk.ac.ebi.scheduler.releaser.model.ReleaseData +import org.springframework.data.repository.kotlin.CoroutineCrudRepository import java.util.Date -interface ReleaserRepository : PagingAndSortingRepository { +interface SubmissionReleaserRepository : CoroutineCrudRepository { @Query( value = "{ released: true, version: { \$gte: 0 } }", fields = "{ accNo: 1, owner: 1, relPath: 1 }" ) - fun findAllReleased(): List + fun findAllReleased(): Flow @Query( value = "{ releaseTime: { \$lte: ?0 }, released: false, version: { \$gte: 0 } }", fields = "{ accNo: 1, owner: 1, relPath: 1 }" ) - fun findAllUntil(toRTime: Date): List + fun findAllUntil(toRTime: Date): Flow @Query( value = "{ releaseTime: { \$gte: ?0, \$lte: ?1 }, released: false, version: { \$gte: 0 } }", fields = "{ accNo: 1, owner: 1, relPath: 1 }" ) - fun findAllBetween(fromRTime: Date, toRTime: Date): List + fun findAllBetween(fromRTime: Date, toRTime: Date): Flow } diff --git a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/SingleFieldDtos.kt b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/SingleFieldDtos.kt index 6e78614905..e8114b1af7 100644 --- a/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/SingleFieldDtos.kt +++ b/submission/persistence-mongo/src/main/kotlin/ac/uk/ebi/biostd/persistence/doc/db/repositories/SingleFieldDtos.kt @@ -8,3 +8,9 @@ import ac.uk.ebi.biostd.persistence.doc.model.DocCollection data class SubmissionCollections(val collections: List?) data class MigrationData(val accNo: String) + +data class ReleaseData( + val accNo: String, + val owner: String, + val relPath: String +) diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt index 19c961151e..8462c70571 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt @@ -5,7 +5,6 @@ import ac.uk.ebi.biostd.submission.domain.request.SubmissionRequestReleaser import ac.uk.ebi.biostd.submission.model.ReleaseRequest import ac.uk.ebi.biostd.submission.web.handlers.SubmissionsWebHandler import ebi.ac.uk.security.integration.model.api.SecurityUser -import kotlinx.coroutines.runBlocking import org.springframework.security.access.prepost.PreAuthorize import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.PathVariable @@ -24,8 +23,8 @@ class SubmissionOperationsResource( private val submissionReleaser: SubmissionRequestReleaser, ) { @PostMapping("/ftp/generate") - fun generateFtpLinks(@RequestParam("accNo", required = true) accNo: String) { - runBlocking { submissionReleaser.generateFtp(accNo) } + suspend fun generateFtpLinks(@RequestParam("accNo", required = true) accNo: String) { + submissionReleaser.generateFtp(accNo) } @DeleteMapping("/{accNo}") From a36ea14b72ed9d3fa221ef09ba104ba90ea4d6d7 Mon Sep 17 00:00:00 2001 From: Jhoan Munoz Date: Sat, 25 Nov 2023 14:23:11 +0000 Subject: [PATCH 2/3] Pivotal ID # 186425798: Regenerate all PMC submission FTP - Fix client operation - Generate links only for released submissions --- .../ebi/biostd/client/api/CommonOperationsClient.kt | 2 +- .../domain/request/SubmissionRequestReleaser.kt | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt index bc3d4fc8e8..1bc1e02385 100644 --- a/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt +++ b/client/bio-webclient/src/main/kotlin/ac/uk/ebi/biostd/client/api/CommonOperationsClient.kt @@ -28,7 +28,7 @@ class CommonOperationsClient( } override suspend fun generateFtpLinks(accNo: String) { - val body = linkedMultiValueMapOf("accNo" to listOf(accNo)) + val body = linkedMultiValueMapOf("accNo" to accNo) client.postAsync("$FTP_URL/generate", RequestParams(body = body)) } diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt index 3610952527..31d1a30cf2 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt @@ -81,11 +81,11 @@ class SubmissionRequestReleaser( */ suspend fun releaseSubmission(accNo: String) { val submission = queryService.getExtByAccNo(accNo, includeFileListFiles = true) - releaseSubmission(submission) + if (submission.released) releaseSubmission(submission) } /** - * Generates/refresh FTP status for a given submission. + * Generates/refresh FTP links for a given submission. */ suspend fun generateFtp(accNo: String) { val sub = queryService.getExtByAccNo(accNo, includeFileListFiles = true) @@ -99,12 +99,16 @@ class SubmissionRequestReleaser( return releasedFile } - private suspend fun releaseSubmission(sub: ExtSubmission) { + private suspend fun releaseSubmissionFiles(sub: ExtSubmission) { logger.info { "${sub.accNo} ${sub.owner} Started releasing submission files over ${sub.storageMode}" } serializationService.filesFlow(sub) .filterNot { it is FireFile && it.published } .collectIndexed { idx, file -> releaseFile(sub, idx, file) } - persistenceService.setAsReleased(sub.accNo) logger.info { "${sub.accNo} ${sub.owner} Finished releasing submission files over ${sub.storageMode}" } } + + private suspend fun releaseSubmission(sub: ExtSubmission) { + releaseSubmissionFiles(sub) + persistenceService.setAsReleased(sub.accNo) + } } From 70448f0448350af9324337c84e52df4cc345748a Mon Sep 17 00:00:00 2001 From: Jhoan Munoz Date: Sun, 26 Nov 2023 18:11:33 +0000 Subject: [PATCH 3/3] Pivotal ID # 186425798: Regenerate all PMC submission FTP - Fix condition - Required submission to be released for generating the ftp links --- .../domain/request/SubmissionRequestReleaser.kt | 10 ++++++---- .../InvalidPublicSubmissionException.kt | 2 ++ .../request/SubmissionRequestReleaserTest.kt | 16 ++++++++++++++++ .../resources/SubmissionOperationsResource.kt | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt index 31d1a30cf2..daabfada3c 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaser.kt @@ -8,6 +8,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestFilesPersistenceService import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService +import ac.uk.ebi.biostd.submission.exceptions.UnreleasedSubmissionException import ebi.ac.uk.extended.model.ExtFile import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.FireFile @@ -81,15 +82,16 @@ class SubmissionRequestReleaser( */ suspend fun releaseSubmission(accNo: String) { val submission = queryService.getExtByAccNo(accNo, includeFileListFiles = true) - if (submission.released) releaseSubmission(submission) + releaseSubmission(submission) } /** * Generates/refresh FTP links for a given submission. */ - suspend fun generateFtp(accNo: String) { - val sub = queryService.getExtByAccNo(accNo, includeFileListFiles = true) - releaseSubmission(sub) + suspend fun generateFtpLinks(accNo: String) { + val submission = queryService.getExtByAccNo(accNo, includeFileListFiles = true) + require(submission.released) { throw UnreleasedSubmissionException() } + releaseSubmissionFiles(submission) } private suspend fun releaseFile(sub: ExtSubmission, idx: Int, file: ExtFile): ExtFile { diff --git a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/exceptions/InvalidPublicSubmissionException.kt b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/exceptions/InvalidPublicSubmissionException.kt index a839f6a8a8..43647a4380 100644 --- a/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/exceptions/InvalidPublicSubmissionException.kt +++ b/submission/submission-core/src/main/kotlin/ac/uk/ebi/biostd/submission/exceptions/InvalidPublicSubmissionException.kt @@ -3,3 +3,5 @@ package ac.uk.ebi.biostd.submission.exceptions class PastReleaseDateException : RuntimeException("Release date cannot be in the past") class InvalidReleaseDateException : RuntimeException("The release date of a public study cannot be changed") + +class UnreleasedSubmissionException : RuntimeException("Can't generate FTP links for a private submission") diff --git a/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaserTest.kt b/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaserTest.kt index 6736e67703..6ba311e6f9 100644 --- a/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaserTest.kt +++ b/submission/submission-core/src/test/kotlin/ac/uk/ebi/biostd/submission/domain/request/SubmissionRequestReleaserTest.kt @@ -9,6 +9,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestFilesPersist import ac.uk.ebi.biostd.persistence.common.service.SubmissionRequestPersistenceService import ac.uk.ebi.biostd.persistence.filesystem.api.FileStorageService import ac.uk.ebi.biostd.submission.common.TEST_CONCURRENCY +import ac.uk.ebi.biostd.submission.exceptions.UnreleasedSubmissionException import ebi.ac.uk.extended.model.ExtSubmission import ebi.ac.uk.extended.model.FireFile import ebi.ac.uk.extended.model.NfsFile @@ -25,8 +26,10 @@ import io.mockk.verify import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows import org.junit.jupiter.api.extension.ExtendWith import uk.ac.ebi.extended.serialization.service.ExtSerializationService @@ -115,4 +118,17 @@ class SubmissionRequestReleaserTest( requestService.saveSubmissionRequest(rqt.withNewStatus(CHECK_RELEASED)) } } + + @Test + fun `generate ftp links for private submission`( + @MockK submission: ExtSubmission, + ) = runTest { + every { submission.released } returns false + coEvery { queryService.getExtByAccNo("S-BSST0", includeFileListFiles = true) } returns submission + + val exception = assertThrows { testInstance.generateFtpLinks("S-BSST0") } + assertThat(exception.message).isEqualTo("Can't generate FTP links for a private submission") + + coVerify(exactly = 0) { storageService.releaseSubmissionFile(any(), any(), any()) } + } } diff --git a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt index 8462c70571..beb449e471 100644 --- a/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt +++ b/submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/web/resources/SubmissionOperationsResource.kt @@ -24,7 +24,7 @@ class SubmissionOperationsResource( ) { @PostMapping("/ftp/generate") suspend fun generateFtpLinks(@RequestParam("accNo", required = true) accNo: String) { - submissionReleaser.generateFtp(accNo) + submissionReleaser.generateFtpLinks(accNo) } @DeleteMapping("/{accNo}")