Pivotal ID # 177413307: Report Submissions With Errors (#368)

* Pivotal ID # 177413307: Report Submissions With Errors - Slack notification for submission with errors at processing level - Create a new queue to log the problems in order to be used for later verification
EBIBioStudies · Apr 12, 2021 · 096005e · 096005e
1 parent c85b64f
commit 096005e
Show file tree

Hide file tree

Showing 17 changed files with 256 additions and 12 deletions.
diff --git a/commons/commons-http/gradle.properties b/commons/commons-http/gradle.properties
@@ -1 +1 @@
-coverage=0.84
+coverage=0.41
diff --git a/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/slack/NotificationsSender.kt b/commons/commons-http/src/main/kotlin/ebi/ac/uk/commons/http/slack/NotificationsSender.kt
@@ -1,5 +1,8 @@
 package ebi.ac.uk.commons.http.slack
 
+import org.springframework.http.HttpEntity
+import org.springframework.http.HttpHeaders
+import org.springframework.http.MediaType.APPLICATION_JSON
 import org.springframework.web.client.RestTemplate
 import org.springframework.web.client.postForEntity
 
@@ -8,6 +11,11 @@ class NotificationsSender(
     private val notificationUrl: String
 ) {
     fun send(notification: SystemNotification) {
-        restTemplate.postForEntity<String>(url = notificationUrl, request = notification.asNotification())
+        val headers = HttpHeaders().apply {
+            accept = listOf(APPLICATION_JSON)
+            contentType = APPLICATION_JSON
+        }
+
+        restTemplate.postForEntity<String>(notificationUrl, HttpEntity(notification.asNotification(), headers))
     }
 }
diff --git a/commons/commons-http/src/test/kotlin/ebi/ac/uk/commons/http/slack/NotificationsSenderTest.kt b/commons/commons-http/src/test/kotlin/ebi/ac/uk/commons/http/slack/NotificationsSenderTest.kt
@@ -0,0 +1,41 @@
+package ebi.ac.uk.commons.http.slack
+
+import io.mockk.every
+import io.mockk.impl.annotations.MockK
+import io.mockk.junit5.MockKExtension
+import io.mockk.slot
+import io.mockk.verify
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.http.HttpEntity
+import org.springframework.http.MediaType.APPLICATION_JSON
+import org.springframework.http.ResponseEntity
+import org.springframework.web.client.RestTemplate
+import org.springframework.web.client.postForEntity
+import java.util.Optional
+
+@ExtendWith(MockKExtension::class)
+class NotificationsSenderTest(
+    @MockK private val restTemplate: RestTemplate
+) {
+    private val testInstance = NotificationsSender(restTemplate, "http://notifications:8080")
+
+    @Test
+    fun send() {
+        val requestSlot = slot<HttpEntity<Notification>>()
+        val notification = Report("system", "subsystem", "result")
+
+        every {
+            restTemplate.postForEntity<String>("http://notifications:8080", capture(requestSlot))
+        } returns ResponseEntity.of(Optional.of(""))
+
+        testInstance.send(notification)
+
+        val request = requestSlot.captured
+        assertThat(request.headers.contentType).isEqualTo(APPLICATION_JSON)
+        assertThat(request.headers.accept).containsExactly(APPLICATION_JSON)
+        assertThat(request.body).isEqualTo(notification.asNotification())
+        verify(exactly = 1) { restTemplate.postForEntity<String>("http://notifications:8080", request) }
+    }
+}
diff --git a/...odel-extended/src/main/kotlin/ebi/ac/uk/extended/events/FailedSubmissionRequestMessage.kt b/...odel-extended/src/main/kotlin/ebi/ac/uk/extended/events/FailedSubmissionRequestMessage.kt
@@ -0,0 +1,21 @@
+package ebi.ac.uk.extended.events
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import ebi.ac.uk.extended.model.FileMode
+
+data class FailedSubmissionRequestMessage(
+    @JsonProperty("accNo")
+    val accNo: String,
+
+    @JsonProperty("version")
+    val version: Int,
+
+    @JsonProperty("fileMode")
+    val fileMode: FileMode,
+
+    @JsonProperty("draftKey")
+    val draftKey: String?,
+
+    @JsonProperty("errorMessage")
+    val errorMessage: String?
+)
diff --git a/...mons-model-extended/src/main/kotlin/ebi/ac/uk/extended/events/SubmissionRequestMessage.kt b/...mons-model-extended/src/main/kotlin/ebi/ac/uk/extended/events/SubmissionRequestMessage.kt
@@ -3,7 +3,7 @@ package ebi.ac.uk.extended.events
 import com.fasterxml.jackson.annotation.JsonProperty
 import ebi.ac.uk.extended.model.FileMode
 
-class SubmissionRequestMessage(
+data class SubmissionRequestMessage(
     @JsonProperty("accNo")
     val accNo: String,
 

diff --git a/events/events-publisher/src/main/kotlin/uk/ac/ebi/events/config/EventsPublisherConfig.kt b/events/events-publisher/src/main/kotlin/uk/ac/ebi/events/config/EventsPublisherConfig.kt
@@ -9,6 +9,7 @@ const val BIOSTUDIES_EXCHANGE = "biostudies-exchange"
 const val SUBMISSIONS_ROUTING_KEY = "bio.submission.published"
 const val SUBMISSIONS_RELEASE_ROUTING_KEY = "bio.submission.published.notification"
 const val SUBMISSIONS_REQUEST_ROUTING_KEY = "bio.submission.requested"
+const val SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY = "bio.submission.failed"
 const val SECURITY_NOTIFICATIONS_ROUTING_KEY = "bio.security.notification"
 
 class EventsPublisherConfig(

diff --git a/events/events-publisher/src/main/kotlin/uk/ac/ebi/events/service/EventsPublisherService.kt b/events/events-publisher/src/main/kotlin/uk/ac/ebi/events/service/EventsPublisherService.kt
@@ -1,5 +1,6 @@
 package uk.ac.ebi.events.service
 
+import ebi.ac.uk.extended.events.FailedSubmissionRequestMessage
 import ebi.ac.uk.extended.events.SecurityNotification
 import ebi.ac.uk.extended.events.SubmissionMessage
 import ebi.ac.uk.extended.model.ExtSubmission
@@ -8,6 +9,7 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate
 import uk.ac.ebi.events.config.BIOSTUDIES_EXCHANGE
 import uk.ac.ebi.events.config.EventsProperties
 import uk.ac.ebi.events.config.SECURITY_NOTIFICATIONS_ROUTING_KEY
+import uk.ac.ebi.events.config.SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY
 import uk.ac.ebi.events.config.SUBMISSIONS_RELEASE_ROUTING_KEY
 import uk.ac.ebi.events.config.SUBMISSIONS_ROUTING_KEY
 import java.time.OffsetDateTime
@@ -27,6 +29,9 @@ class EventsPublisherService(
         rabbitTemplate.convertAndSend(
             BIOSTUDIES_EXCHANGE, SUBMISSIONS_RELEASE_ROUTING_KEY, submissionMessage(submission))
 
+    fun submissionFailed(request: FailedSubmissionRequestMessage) =
+        rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY, request)
+
     private fun submissionMessage(submission: ExtSubmission) =
         SubmissionMessage(
             accNo = submission.accNo,

diff --git a/...s/events-publisher/src/test/kotlin/uk/ac/ebi/events/service/EventsPublisherServiceTest.kt b/...s/events-publisher/src/test/kotlin/uk/ac/ebi/events/service/EventsPublisherServiceTest.kt
@@ -1,5 +1,6 @@
 package uk.ac.ebi.events.service
 
+import ebi.ac.uk.extended.events.FailedSubmissionRequestMessage
 import ebi.ac.uk.extended.events.SecurityNotification
 import ebi.ac.uk.extended.events.SubmissionMessage
 import ebi.ac.uk.extended.model.ExtSubmission
@@ -20,6 +21,7 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate
 import uk.ac.ebi.events.config.BIOSTUDIES_EXCHANGE
 import uk.ac.ebi.events.config.EventsProperties
 import uk.ac.ebi.events.config.SECURITY_NOTIFICATIONS_ROUTING_KEY
+import uk.ac.ebi.events.config.SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY
 import uk.ac.ebi.events.config.SUBMISSIONS_RELEASE_ROUTING_KEY
 import uk.ac.ebi.events.config.SUBMISSIONS_ROUTING_KEY
 import java.time.OffsetDateTime
@@ -104,4 +106,17 @@ class EventsPublisherServiceTest(
             rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, SUBMISSIONS_RELEASE_ROUTING_KEY, notification)
         }
     }
+
+    @Test
+    fun submissionFailed(@MockK request: FailedSubmissionRequestMessage) {
+        every {
+            rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY, request)
+        } answers { nothing }
+
+        testInstance.submissionFailed(request)
+
+        verify(exactly = 1) {
+            rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY, request)
+        }
+    }
 }
diff --git a/...ssion-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/NotificationProperties.kt b/...ssion-config/src/main/kotlin/ac/uk/ebi/biostd/common/properties/NotificationProperties.kt
@@ -7,6 +7,7 @@ import java.util.Properties
 class NotificationProperties {
     lateinit var smtp: String
     lateinit var uiUrl: String
+    lateinit var slackUrl: String
 
     @NestedConfigurationProperty
     var rt: RtConfig = RtConfig()

diff --git a/submission/submission-handlers/build.gradle.kts b/submission/submission-handlers/build.gradle.kts
@@ -17,6 +17,7 @@ plugins {
 dependencies {
     api(project(":commons:commons-model-extended"))
     api(project(":commons:commons-model-extended-serialization"))
+    api(project(":commons:commons-http"))
     api(project(":commons:commons-util"))
     api(project(":submission:notifications"))
     api(project(":submission:submission-config"))

diff --git a/...submission-handlers/src/main/kotlin/ac/uk/ebi/biostd/handlers/config/ApplicationConfig.kt b/...submission-handlers/src/main/kotlin/ac/uk/ebi/biostd/handlers/config/ApplicationConfig.kt
@@ -6,6 +6,7 @@ import ac.uk.ebi.biostd.handlers.listeners.SecurityNotificationListener
 import ac.uk.ebi.biostd.handlers.listeners.SubmissionNotificationsListener
 import ac.uk.ebi.biostd.persistence.common.service.NotificationsDataService
 import ac.uk.ebi.biostd.persistence.integration.config.NotificationPersistenceConfig
+import ebi.ac.uk.commons.http.slack.NotificationsSender
 import ebi.ac.uk.notifications.integration.NotificationConfig
 import ebi.ac.uk.notifications.service.RtNotificationService
 import ebi.ac.uk.notifications.service.SecurityNotificationService
@@ -25,10 +26,14 @@ class Listeners {
     @Bean
     fun notificationsListener(
         webConsumer: BioStudiesWebConsumer,
+        notificationsSender: NotificationsSender,
         applicationProperties: ApplicationProperties,
         rtNotificationService: RtNotificationService
-    ): SubmissionNotificationsListener =
-        SubmissionNotificationsListener(webConsumer, rtNotificationService, applicationProperties.notifications)
+    ): SubmissionNotificationsListener = SubmissionNotificationsListener(
+        webConsumer,
+        notificationsSender,
+        rtNotificationService,
+        applicationProperties.notifications)
 
     @Bean
     fun securityNotificationsListener(
@@ -44,6 +49,13 @@ class Services {
         extSerializationService: ExtSerializationService
     ): BioStudiesWebConsumer = BioStudiesWebConsumer(restTemplate, extSerializationService)
 
+    @Bean
+    fun notificationsSender(
+        restTemplate: RestTemplate,
+        applicationProperties: ApplicationProperties
+    ): NotificationsSender =
+        NotificationsSender(restTemplate, applicationProperties.notifications.slackUrl)
+
     @Bean
     fun extSerializationService(): ExtSerializationService = ExtSerializationService()
 

diff --git a/...sion/submission-handlers/src/main/kotlin/ac/uk/ebi/biostd/handlers/config/QueuesConfig.kt b/...sion/submission-handlers/src/main/kotlin/ac/uk/ebi/biostd/handlers/config/QueuesConfig.kt
@@ -11,10 +11,13 @@ const val BIOSTUDIES_EXCHANGE = "biostudies-exchange"
 const val SUBMISSIONS_ROUTING_KEY = "bio.submission.published"
 const val SUBMISSIONS_RELEASE_ROUTING_KEY = "bio.submission.published.notification"
 const val SECURITY_NOTIFICATIONS_ROUTING_KEY = "bio.security.notification"
+const val SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY = "bio.submission.failed"
 
 const val LOG_QUEUE = "submission-submitted-log-queue"
 const val ST_LOG_QUEUE = "submission-submitted-st-queue"
 const val PARTIALS_QUEUE = "submission-submitted-partials-queue"
+const val FAILED_SUBMISSIONS_LOG_QUEUE = "submission-failed-log-queue"
+const val FAILED_SUBMISSIONS_NOTIFICATIONS_QUEUE = "submission-failed-notifications-queue"
 const val SUBMIT_NOTIFICATIONS_QUEUE = "submission-submitted-notifications-queue"
 const val RELEASE_NOTIFICATIONS_QUEUE = "submission-released-notifications-queue"
 const val SECURITY_NOTIFICATIONS_QUEUE = "security-notifications-queue"
@@ -42,6 +45,12 @@ class QueuesConfig {
     @Bean
     fun partialUpdatesQueue(): Queue = Queue(PARTIALS_QUEUE, DURABLES_QUEUES)
 
+    @Bean
+    fun failedSubmissionsLogQueue(): Queue = Queue(FAILED_SUBMISSIONS_LOG_QUEUE, DURABLES_QUEUES)
+
+    @Bean
+    fun failedSubmissionNotificationsQueue(): Queue = Queue(FAILED_SUBMISSIONS_NOTIFICATIONS_QUEUE, DURABLES_QUEUES)
+
     @Bean
     fun securityNotificationsQueue(): Queue = Queue(SECURITY_NOTIFICATIONS_QUEUE, DURABLES_QUEUES)
 
@@ -65,6 +74,17 @@ class QueuesConfig {
     fun partialUpdatesQueueBinding(exchange: TopicExchange): Binding =
         BindingBuilder.bind(partialUpdatesQueue()).to(exchange).with(SUBMISSIONS_ROUTING_KEY)
 
+    @Bean
+    fun failedSubmissionLogQueueBinding(exchange: TopicExchange): Binding =
+        BindingBuilder.bind(failedSubmissionsLogQueue()).to(exchange).with(SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY)
+
+    @Bean
+    fun failedSubmissionNotificationQueueBinding(exchange: TopicExchange): Binding =
+        BindingBuilder
+            .bind(failedSubmissionNotificationsQueue())
+            .to(exchange)
+            .with(SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY)
+
     @Bean
     fun securityNotificationsQueueBinding(exchange: TopicExchange): Binding =
         BindingBuilder.bind(securityNotificationsQueue()).to(exchange).with(SECURITY_NOTIFICATIONS_ROUTING_KEY)

diff --git a/...rs/src/main/kotlin/ac/uk/ebi/biostd/handlers/listeners/SubmissionNotificationsListener.kt b/...rs/src/main/kotlin/ac/uk/ebi/biostd/handlers/listeners/SubmissionNotificationsListener.kt
@@ -2,17 +2,24 @@ package ac.uk.ebi.biostd.handlers.listeners
 
 import ac.uk.ebi.biostd.common.properties.NotificationProperties
 import ac.uk.ebi.biostd.handlers.api.BioStudiesWebConsumer
+import ac.uk.ebi.biostd.handlers.config.FAILED_SUBMISSIONS_NOTIFICATIONS_QUEUE
 import ac.uk.ebi.biostd.handlers.config.RELEASE_NOTIFICATIONS_QUEUE
 import ac.uk.ebi.biostd.handlers.config.SUBMIT_NOTIFICATIONS_QUEUE
+import ebi.ac.uk.commons.http.slack.Alert
+import ebi.ac.uk.commons.http.slack.NotificationsSender
+import ebi.ac.uk.extended.events.FailedSubmissionRequestMessage
 import ebi.ac.uk.extended.events.SubmissionMessage
 import ebi.ac.uk.notifications.service.RtNotificationService
 import mu.KotlinLogging
 import org.springframework.amqp.rabbit.annotation.RabbitListener
 
 private val logger = KotlinLogging.logger {}
+private const val SYSTEM_NAME = "Submitter"
+private const val HANDLERS_SUBSYSTEM = "Submission Handlers"
 
 class SubmissionNotificationsListener(
     private val webConsumer: BioStudiesWebConsumer,
+    private val notificationsSender: NotificationsSender,
     private val rtNotificationService: RtNotificationService,
     private val notificationProperties: NotificationProperties
 ) {
@@ -37,4 +44,16 @@ class SubmissionNotificationsListener(
             rtNotificationService.notifySubmissionRelease(submission, owner.fullName, notificationProperties.uiUrl)
         }
     }
+
+    @RabbitListener(queues = [FAILED_SUBMISSIONS_NOTIFICATIONS_QUEUE])
+    fun receiveFailedSubmissionMessage(msg: FailedSubmissionRequestMessage) {
+        notificationsSender.send(
+            Alert(
+                SYSTEM_NAME,
+                HANDLERS_SUBSYSTEM,
+                "Problem processing submission '${msg.accNo}' with version ${msg.version} in mode '${msg.fileMode}'",
+                msg.errorMessage
+            )
+        )
+    }
 }
diff --git a/submission/submission-handlers/src/main/resources/application.yml b/submission/submission-handlers/src/main/resources/application.yml
@@ -21,6 +21,7 @@ app:
   notifications:
     smtp: smtp.ebi.ac.uk
     uiUrl: # The BioStudies UI url
+    slackUrl: # Slack notifications URL
     rt:
       host: # RT server host
       queue: # Queue used for RT notifications

diff --git a/...rc/test/kotlin/ac/uk/ebi/biostd/handlers/listeners/SubmissionNotificationsListenerTest.kt b/...rc/test/kotlin/ac/uk/ebi/biostd/handlers/listeners/SubmissionNotificationsListenerTest.kt
@@ -2,14 +2,19 @@ package ac.uk.ebi.biostd.handlers.listeners
 
 import ac.uk.ebi.biostd.common.properties.NotificationProperties
 import ac.uk.ebi.biostd.handlers.api.BioStudiesWebConsumer
+import ebi.ac.uk.commons.http.slack.NotificationsSender
+import ebi.ac.uk.commons.http.slack.SystemNotification
+import ebi.ac.uk.extended.events.FailedSubmissionRequestMessage
 import ebi.ac.uk.extended.events.SubmissionMessage
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.extended.model.ExtUser
+import ebi.ac.uk.extended.model.FileMode.COPY
 import ebi.ac.uk.notifications.service.RtNotificationService
 import io.mockk.clearAllMocks
 import io.mockk.every
 import io.mockk.impl.annotations.MockK
 import io.mockk.junit5.MockKExtension
+import io.mockk.slot
 import io.mockk.verify
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
@@ -22,11 +27,12 @@ class SubmissionNotificationsListenerTest(
     @MockK private val submission: ExtSubmission,
     @MockK private val message: SubmissionMessage,
     @MockK private val webConsumer: BioStudiesWebConsumer,
+    @MockK private val notificationsSender: NotificationsSender,
     @MockK private val rtNotificationService: RtNotificationService,
     @MockK private val notificationProperties: NotificationProperties
 ) {
     private val testInstance =
-        SubmissionNotificationsListener(webConsumer, rtNotificationService, notificationProperties)
+        SubmissionNotificationsListener(webConsumer, notificationsSender, rtNotificationService, notificationProperties)
 
     @BeforeEach
     fun beforeEach() {
@@ -78,6 +84,18 @@ class SubmissionNotificationsListenerTest(
         verify(exactly = 0) { rtNotificationService.notifySubmissionRelease(submission, "Dr Owner", "ui-url") }
     }
 
+    @Test
+    fun `notify failed submission`() {
+        val notificationSlot = slot<SystemNotification>()
+        val message = FailedSubmissionRequestMessage("S-BSST1", 1, COPY, "TMP_123", "error message")
+
+        every { notificationsSender.send(capture(notificationSlot)) } answers { nothing }
+
+        testInstance.receiveFailedSubmissionMessage(message)
+
+        verify(exactly = 1) { notificationsSender.send(notificationSlot.captured) }
+    }
+
     private fun mockMessage() {
         every { message.extTabUrl } returns "ext-tab-url"
         every { message.extUserUrl } returns "ext-user-url"

diff --git a/...on-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt b/...on-webapp/src/main/kotlin/ac/uk/ebi/biostd/submission/domain/service/SubmissionService.kt
@@ -13,6 +13,7 @@ import ac.uk.ebi.biostd.persistence.common.service.SubmissionQueryService
 import ac.uk.ebi.biostd.submission.ext.getSimpleByAccNo
 import ac.uk.ebi.biostd.submission.model.SubmissionRequest
 import ac.uk.ebi.biostd.submission.submitter.SubmissionSubmitter
+import ebi.ac.uk.extended.events.FailedSubmissionRequestMessage
 import ebi.ac.uk.extended.events.SubmissionRequestMessage
 import ebi.ac.uk.extended.model.ExtSubmission
 import ebi.ac.uk.security.integration.components.IUserPrivilegesService
@@ -59,11 +60,19 @@ class SubmissionService(
     fun processSubmission(request: SubmissionRequestMessage) {
         logger.info { "received process message for submission ${request.accNo} , version: ${request.version}" }
 
-        val submission = getRequest(request.accNo, request.version)
-        val saveRequest = SaveSubmissionRequest(submission, request.fileMode, request.draftKey)
-        val processed = submissionSubmitter.processRequest(saveRequest)
+        runCatching {
+            val submission = getRequest(request.accNo, request.version)
+            val saveRequest = SaveSubmissionRequest(submission, request.fileMode, request.draftKey)
+            val processed = submissionSubmitter.processRequest(saveRequest)
 
-        eventsPublisherService.submissionSubmitted(processed)
+            eventsPublisherService.submissionSubmitted(processed)
+        }.onFailure {
+            val (accNo, version, fileMode, draftKey) = request
+            val message = FailedSubmissionRequestMessage(accNo, version, fileMode, draftKey, it.message)
+
+            logger.error { "Problem processing submission request '$accNo': ${it.message}" }
+            eventsPublisherService.submissionFailed(message)
+        }
     }
 
     fun getSubmissionAsJson(accNo: String): String =