Pivotal ID # 171382047: Automate Stats Publishing

commons/commons-model-extended-serialization/src/main/kotlin/uk/ac/ebi/extended/serialization/service/ExtSerializationServiceExt.kt

@@ -6,7 +6,7 @@ import ebi.ac.uk.extended.model.allFileList
 import ebi.ac.uk.extended.model.allSectionsFiles
 
 /**
- * Return a sequence with all file list of a submission. Pagetab files are retrieved first, followed by section files
+ * Return a sequence with all the files of a submission. Pagetab files are retrieved first, followed by section files
  * and file list files.
  */
 fun ExtSerializationService.fileSequence(submission: ExtSubmission): Sequence<ExtFile> {

commons/commons-model-extended/src/main/kotlin/ebi/ac/uk/extended/events/RequestCreated.kt

@@ -41,3 +41,8 @@ data class RequestPersisted(
     @JsonProperty("accNo") override val accNo: String,
     @JsonProperty("version") override val version: Int,
 ) : RequestMessage
+
+data class RequestFinalized(
+    @JsonProperty("accNo") override val accNo: String,
+    @JsonProperty("version") override val version: Int,
+) : RequestMessage

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/CopyFileVisitor.kt

@@ -1,7 +1,6 @@
 package ebi.ac.uk.io
 
 import java.nio.file.FileVisitResult
-import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
@@ -18,8 +17,7 @@ internal class CopyFileVisitor(
 
     override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
         val target = targetPath.resolve(sourcePath.relativize(file))
-        Files.copy(file, target)
-        Files.setPosixFilePermissions(target, permissions.file)
+        FileUtilsHelper.copyFile(file, target, permissions)
 
         return FileVisitResult.CONTINUE
     }

commons/commons-util/src/main/kotlin/ebi/ac/uk/io/FileUtils.kt

@@ -216,8 +216,10 @@ internal object FileUtilsHelper {
         target: Path,
         permissions: Permissions,
     ) {
-        Files.copy(source, createParentDirectories(target, permissions.folder), REPLACE_EXISTING)
-        Files.setPosixFilePermissions(target, permissions.file)
+        runSafely {
+            Files.copy(source, createParentDirectories(target, permissions.folder), REPLACE_EXISTING)
+            Files.setPosixFilePermissions(target, permissions.file)
+        }
     }
 
     fun moveFile(
@@ -244,16 +246,21 @@ internal object FileUtilsHelper {
 
         return directoryPath
     }
+    private fun createDirectory(path: Path, permissions: Set<PosixFilePermission>) {
+        runSafely {
+            Files.createDirectory(path)
+            Files.setPosixFilePermissions(path, permissions)
+        }
+    }
 
     /*
-     * This method requires ignoring exceptions of the type FileAlreadyExistsException in order to become thread safe.
+     * Some methods require ignoring exceptions of the type FileAlreadyExistsException in order to become thread safe.
      * See this discussion for more details:
      * https://github.com/EBIBioStudies/biostudies-backend-services/pull/733#discussion_r1280085979
      */
-    private fun createDirectory(path: Path, permissions: Set<PosixFilePermission>) {
+    private fun runSafely(func: () -> Unit) {
         runCatching {
-            Files.createDirectory(path)
-            Files.setPosixFilePermissions(path, permissions)
+            func()
         }.onFailure {
             if ((it is FileAlreadyExistsException).not()) throw it
         }

events/events-publisher/src/main/kotlin/uk/ac/ebi/events/service/EventsPublisherService.kt

@@ -10,6 +10,7 @@ import ebi.ac.uk.extended.events.RequestCheckedReleased
 import ebi.ac.uk.extended.events.RequestCleaned
 import ebi.ac.uk.extended.events.RequestCreated
 import ebi.ac.uk.extended.events.RequestFilesCopied
+import ebi.ac.uk.extended.events.RequestFinalized
 import ebi.ac.uk.extended.events.RequestIndexed
 import ebi.ac.uk.extended.events.RequestLoaded
 import ebi.ac.uk.extended.events.RequestMessage
@@ -69,6 +70,11 @@ class EventsPublisherService(
             BIOSTUDIES_EXCHANGE, notificationsProperties.requestRoutingKey, RequestCreated(accNo, version)
         )
 
+    fun submissionFinalized(accNo: String, version: Int) =
+        rabbitTemplate.convertAndSend(
+            BIOSTUDIES_EXCHANGE, notificationsProperties.requestRoutingKey, RequestFinalized(accNo, version)
+        )
+
     fun submissionSubmitted(accNo: String, owner: String) =
         rabbitTemplate.convertAndSend(
             BIOSTUDIES_EXCHANGE, SUBMISSIONS_ROUTING_KEY, submissionMessage(accNo, owner)

events/events-publisher/src/test/kotlin/uk/ac/ebi/events/service/EventsPublisherServiceTest.kt

@@ -6,6 +6,7 @@ import ac.uk.ebi.biostd.common.events.SUBMISSIONS_FAILED_REQUEST_ROUTING_KEY
 import ac.uk.ebi.biostd.common.events.SUBMISSIONS_ROUTING_KEY
 import ac.uk.ebi.biostd.common.properties.NotificationsProperties
 import ebi.ac.uk.extended.events.RequestCleaned
+import ebi.ac.uk.extended.events.RequestFinalized
 import ebi.ac.uk.extended.events.RequestMessage
 import ebi.ac.uk.extended.events.RequestPersisted
 import ebi.ac.uk.extended.events.SecurityNotification
@@ -123,6 +124,20 @@ class EventsPublisherServiceTest(
         verify(exactly = 1) { rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, rKey, request) }
     }
 
+    @Test
+    fun `request finalized`() {
+        val requestSlot = slot<RequestFinalized>()
+        every { notificationsProperties.requestRoutingKey } returns rKey
+        every { rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, rKey, capture(requestSlot)) } answers { nothing }
+
+        testInstance.submissionFinalized("S-BSST0", 1)
+
+        val request = requestSlot.captured
+        assertThat(request.version).isEqualTo(1)
+        assertThat(request.accNo).isEqualTo("S-BSST0")
+        verify(exactly = 1) { rabbitTemplate.convertAndSend(BIOSTUDIES_EXCHANGE, rKey, request) }
+    }
+
     private companion object {
         const val rKey = "key"
     }

submission/submission-webapp/src/itest/kotlin/ac/uk/ebi/biostd/itest/test/stats/SubmissionStatsTest.kt

@@ -0,0 +1,197 @@
+package ac.uk.ebi.biostd.itest.test.stats
+
+import ac.uk.ebi.biostd.client.integration.commons.SubmissionFormat.TSV
+import ac.uk.ebi.biostd.client.integration.web.BioWebClient
+import ac.uk.ebi.biostd.common.config.FilePersistenceConfig
+import ac.uk.ebi.biostd.itest.common.SecurityTestService
+import ac.uk.ebi.biostd.itest.entities.RegularUser
+import ac.uk.ebi.biostd.itest.entities.SuperUser
+import ac.uk.ebi.biostd.itest.itest.ITestListener.Companion.tempFolder
+import ac.uk.ebi.biostd.itest.itest.getWebClient
+import ac.uk.ebi.biostd.persistence.common.model.SubmissionStatType.FILES_SIZE
+import ac.uk.ebi.biostd.persistence.common.service.StatsDataService
+import ac.uk.ebi.biostd.persistence.common.service.SubmissionPersistenceQueryService
+import ebi.ac.uk.asserts.assertThat
+import ebi.ac.uk.dsl.tsv.line
+import ebi.ac.uk.dsl.tsv.tsv
+import ebi.ac.uk.extended.model.StorageMode.NFS
+import ebi.ac.uk.io.ext.createFile
+import ebi.ac.uk.util.date.toStringDate
+import org.assertj.core.api.Assertions.assertThat
+import org.awaitility.Awaitility.await
+import org.awaitility.Durations.TEN_SECONDS
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty
+import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.test.context.SpringBootTest
+import org.springframework.boot.test.web.server.LocalServerPort
+import org.springframework.context.annotation.Import
+import org.springframework.test.context.junit.jupiter.SpringExtension
+import java.time.OffsetDateTime
+
+@Import(FilePersistenceConfig::class)
+@ExtendWith(SpringExtension::class)
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+class SubmissionStatsTest(
+    @Autowired val statsDataService: StatsDataService,
+    @Autowired val securityTestService: SecurityTestService,
+    @Autowired val submissionRepository: SubmissionPersistenceQueryService,
+    @LocalServerPort val serverPort: Int,
+) {
+    private lateinit var webClient: BioWebClient
+
+    @BeforeAll
+    fun init() {
+        securityTestService.ensureUserRegistration(SuperUser)
+        securityTestService.ensureUserRegistration(RegularUser)
+        webClient = getWebClient(serverPort, SuperUser)
+    }
+
+    @Test
+    @EnabledIfSystemProperty(named = "enableFire", matches = "true")
+    fun `26-1 files size stat calculation on submit over FIRE`() {
+        val version1 = tsv {
+            line("Submission", "S-STTS1")
+            line("Title", "Stats Registration Test Over FIRE")
+            line("ReleaseDate", OffsetDateTime.now().toStringDate())
+            line()
+
+            line("Study")
+            line()
+
+            line("File", "stats file 1.doc")
+            line("Type", "test")
+            line()
+        }.toString()
+
+        val version2 = tsv {
+            line("Submission", "S-STTS1")
+            line("Title", "Stats Registration Test Over FIRE")
+            line("ReleaseDate", OffsetDateTime.now().toStringDate())
+            line()
+
+            line("Study")
+            line("Type", "Experiment")
+            line("File List", "file-list.tsv")
+            line()
+
+            line("File", "stats file 1.doc")
+            line("Type", "test")
+            line()
+
+            line("Experiment", "Exp1")
+            line("Type", "Subsection")
+            line()
+
+            line("File", "statsFile2.txt")
+            line("Type", "Attached")
+            line()
+        }.toString()
+
+        val fileListContent = tsv {
+            line("Files", "Type")
+            line("a/statsFile3.pdf", "inner")
+            line("a", "folder")
+        }.toString()
+
+        webClient.uploadFiles(
+            listOf(
+                tempFolder.createFile("statsFile2.txt", "content"),
+                tempFolder.createFile("file-list.tsv", fileListContent),
+                tempFolder.createFile("stats file 1.doc", "doc content"),
+            )
+        )
+        webClient.uploadFiles(listOf(tempFolder.createFile("statsFile3.pdf", "pdf content")), "a")
+
+        assertThat(webClient.submitSingle(version1, TSV)).isSuccessful()
+        await().atMost(TEN_SECONDS).until { statsDataService.findByAccNo("S-STTS1").isNotEmpty() }
+        val statVersion1 = statsDataService.findByAccNo("S-STTS1")
+        assertThat(statVersion1).hasSize(1)
+        assertThat(statVersion1.first().value).isEqualTo(1211L)
+        assertThat(statVersion1.first().type).isEqualTo(FILES_SIZE)
+        assertThat(statVersion1.first().accNo).isEqualTo("S-STTS1")
+
+        assertThat(webClient.submitSingle(version2, TSV)).isSuccessful()
+        await().atMost(TEN_SECONDS).until { statsDataService.findByAccNo("S-STTS1").first().value != 1211L }
+        val stats = statsDataService.findByAccNo("S-STTS1")
+        assertThat(stats).hasSize(1)
+        assertThat(stats.first().value).isEqualTo(3529L)
+        assertThat(stats.first().type).isEqualTo(FILES_SIZE)
+        assertThat(stats.first().accNo).isEqualTo("S-STTS1")
+    }
+
+    @Test
+    @EnabledIfSystemProperty(named = "enableFire", matches = "false")
+    fun `26-2 files size stat calculation on submit over NFS`() {
+        val version1 = tsv {
+            line("Submission", "S-STTS2")
+            line("Title", "Stats Registration Test Over NFS")
+            line("ReleaseDate", OffsetDateTime.now().toStringDate())
+            line()
+
+            line("Study")
+            line()
+
+            line("File", "stats file 1.doc")
+            line("Type", "test")
+            line()
+        }.toString()
+
+        val version2 = tsv {
+            line("Submission", "S-STTS2")
+            line("Title", "Stats Registration Test Over NFS")
+            line("ReleaseDate", OffsetDateTime.now().toStringDate())
+            line()
+
+            line("Study")
+            line("Type", "Experiment")
+            line("File List", "file-list.tsv")
+            line()
+
+            line("File", "stats file 1.doc")
+            line("Type", "test")
+            line()
+
+            line("Experiment", "Exp1")
+            line("Type", "Subsection")
+            line()
+
+            line("File", "statsFile2.txt")
+            line("Type", "Attached")
+            line()
+        }.toString()
+
+        val fileListContent = tsv {
+            line("Files", "Type")
+            line("a/statsFile3.pdf", "inner")
+            line("a", "folder")
+        }.toString()
+
+        webClient.uploadFiles(
+            listOf(
+                tempFolder.createFile("statsFile2.txt", "content"),
+                tempFolder.createFile("file-list.tsv", fileListContent),
+                tempFolder.createFile("stats file 1.doc", "doc content"),
+            )
+        )
+        webClient.uploadFiles(listOf(tempFolder.createFile("statsFile3.pdf", "pdf content")), "a")
+
+        assertThat(webClient.submitSingle(version1, TSV, NFS)).isSuccessful()
+        await().atMost(TEN_SECONDS).until { statsDataService.findByAccNo("S-STTS2").isNotEmpty() }
+        val statVersion1 = statsDataService.findByAccNo("S-STTS2")
+        assertThat(statVersion1).hasSize(1)
+        assertThat(statVersion1.first().value).isEqualTo(1208L)
+        assertThat(statVersion1.first().type).isEqualTo(FILES_SIZE)
+        assertThat(statVersion1.first().accNo).isEqualTo("S-STTS2")
+
+        assertThat(webClient.submitSingle(version2, TSV, NFS)).isSuccessful()
+        await().atMost(TEN_SECONDS).until { statsDataService.findByAccNo("S-STTS2").first().value != 1208L }
+        val stats = statsDataService.findByAccNo("S-STTS2")
+        assertThat(stats).hasSize(1)
+        assertThat(stats.first().value).isEqualTo(3364L)
+        assertThat(stats.first().type).isEqualTo(FILES_SIZE)
+        assertThat(stats.first().accNo).isEqualTo("S-STTS2")
+    }
+}

submission/submission-webapp/src/itest/resources/itestsInventory/itestsInventory.md

@@ -130,4 +130,6 @@
 | SubmissionRefreshApiTest        | 25-1    | Refresh when submission title is updated                              |                                                                                                                 |
 | SubmissionRefreshApiTest        | 25-2    | Refresh when submission release date is updated                       |                                                                                                                 |
 | SubmissionRefreshApiTest        | 25-3    | Refresh when submission attribute is updated                          |                                                                                                                 |
-| SubmissionRefreshApiTest        | 25-4    | Refresh when submission fileListFile attribute is updated             |                                                                                                                 |                                                                      |
+| SubmissionRefreshApiTest        | 25-4    | Refresh when submission fileListFile attribute is updated             |                                                                                                                 |                                                                      |
+| SubmissionStatsTest             | 26-1    | files size stat calculation on submit over FIRE                       |                                                                                                                 |                                                                      |
+| SubmissionStatsTest             | 26-2    | files size stat calculation on submit over NFS                        |                                                                                                                 |                                                                      |

submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmissionConfig.kt

@@ -39,6 +39,7 @@ import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Import
 import uk.ac.ebi.events.service.EventsPublisherService
+import uk.ac.ebi.extended.serialization.service.ExtSerializationService
 import java.net.URI
 
 @Configuration
@@ -84,16 +85,25 @@ class SubmissionConfig(
 
     @Bean
     fun submissionStagesHandler(
+        statsService: SubmissionStatsService,
         submissionSubmitter: SubmissionSubmitter,
         eventsPublisherService: EventsPublisherService,
-    ): SubmissionStagesHandler = SubmissionStagesHandler(submissionSubmitter, eventsPublisherService)
+    ): SubmissionStagesHandler = SubmissionStagesHandler(statsService, submissionSubmitter, eventsPublisherService)
 
     @Bean
     fun submissionStatsService(
         statsFileHandler: StatsFileHandler,
         tempFileGenerator: TempFileGenerator,
         submissionStatsService: StatsDataService,
-    ): SubmissionStatsService = SubmissionStatsService(statsFileHandler, tempFileGenerator, submissionStatsService)
+        extSerializationService: ExtSerializationService,
+        extSubmissionQueryService: ExtSubmissionQueryService,
+    ): SubmissionStatsService = SubmissionStatsService(
+        statsFileHandler,
+        tempFileGenerator,
+        submissionStatsService,
+        extSerializationService,
+        extSubmissionQueryService,
+    )
 
     @Bean
     fun extSubmissionQueryService(

submission/submission-webapp/src/main/kotlin/ac/uk/ebi/biostd/common/config/SubmitterConfig.kt

@@ -89,7 +89,7 @@ class SubmitterConfig(
             fileProcessingService,
             persistenceService,
             filesRequestService,
-            eventsPublisherService
+            eventsPublisherService,
         )
     }
 
@@ -142,11 +142,13 @@ class SubmitterConfig(
     fun submissionRequestFinalizer(
         storageService: FileStorageService,
         serializationService: ExtSerializationService,
+        eventsPublisherService: EventsPublisherService,
         queryService: SubmissionPersistenceQueryService,
         requestService: SubmissionRequestPersistenceService,
     ): SubmissionRequestFinalizer = SubmissionRequestFinalizer(
         storageService,
         serializationService,
+        eventsPublisherService,
         queryService,
         requestService,
     )