Promote To DEV (#258)

Promote To DEV

Features:
* Pivotal ID # 172817960: Resubmit Notification (#234)
* Pivotal ID # 173574802: Submissions Files Permissions (#235)
* Pivotal ID # 174054535: Disable SuperUser In Request
* Pivotal ID # 169837866: Async File Processing (#256)
* Pivotal ID # 173792650: Replace Base Path For Multiple Absolute Paths  (#239)
* Pivotal ID # 174127719: Transaction Lock (#250)

Bug Fixes:
* Pivotal ID # 173611760: Improve Stats AccNo Handling (#237)
* Pivotal ID # 173457329: Project Access Tag Validation (#232)
* Pivotal ID # 173569202: Empty Section Parent Cell (#240)
* Pivotal ID # 173213544: User files permissions (#242)
* Pivotal ID # 173457302: RelPath Calculation (#244)
* Pivotal ID # 173750220: Author TSV Serialization (#245)
* Pivotal ID # 174316468: Project Refresh AccNo Template Validation (#257)
* Pivotal ID # 171230259: Draft Creation Times Out For Big Submissions (#251)

Chores:
* Pivotal ID # 172435409: Remove Old Backend Dependencies (#238)
* Pivotal ID # 174076301: Update PMC Api (#248)
* Pivotal ID #174275642: Submission Loading Strategy (#254)

Co-authored-by: Juan <jcamilorada@ebi.ac.uk>

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: gradle:6.0-jdk8
+image: gradle:6.5.1-jdk8
 include:
   - local: '/ci/submitter.yml'
   - local: '/ci/pmc-processor.yml'

README.md

@@ -33,9 +33,22 @@ This section will explain how to deploy a local instance of the submitter webapp
 * Docker
 
 #### Database Setup
-1. Execute the command `gradle setUpTestDatabase` in the [infrastructure](infrastructure) folder. This will deploy a
-custom MySql image loaded with the application's schema and some initial test data.
-2. Execute the command `docker ps`. You should see the _biostudies-mysql_ image up and running.
+Execute the command `gradle setUpTestDatabase` in the [infrastructure](infrastructure) folder. This will deploy a
+custom MySql image loaded with the application's schema and some initial test data and the following configuration:
+* Container name: biostudies-mysql
+* User: root
+* Password: admin
+* Port: 3306
+
+
+#### RabbitMQ Setup
+Execute the command `gradle setUpRabbitMQ` in the [infrastructure](infrastructure) folder. This will deploy a
+RabbitMQ image with the following configuration:
+* Container name: biostudies-rabbitmq
+* User: manager
+* Password: manager-local
+* Port: 5672
+* Admin console: http://localhost:15672
 
 #### Configure WebApp
 1. Move to [Submission WebApp](submission/submission-webapp) folder
@@ -45,6 +58,8 @@ the [Local Submission WebApp Config File](submission/submission-webapp/src/main/
 folder for fresh instances. Set this folder path in the **basepath** property of the configuration file
 4. Execute the command `gradle bootRun`
 
+>Note: In order for the messages in the rabbit queues to be processed you need to start the Handlers application
+
 #### Make A Submission
 In order to make a submission, you can use the BioStudies CLI. Find more information
 [here](client/bio-commandline/README.md).

bio-admin/build.gradle.kts

@@ -5,7 +5,7 @@ import SpringBootDependencies.SpringBootStarterWeb
 import org.springframework.boot.gradle.tasks.bundling.BootJar
 
 plugins {
-    id("org.jetbrains.kotlin.plugin.spring") version "1.3.20"
+    id("org.jetbrains.kotlin.plugin.spring") version "1.3.72"
     id("io.spring.dependency-management") version "1.0.6.RELEASE"
     id("org.springframework.boot") version "2.1.2.RELEASE"
 }

build.gradle.kts

@@ -1,6 +1,6 @@
 plugins {
     id("io.gitlab.arturbosch.detekt") version "1.3.1"
-    id("org.jetbrains.kotlin.jvm") version "1.3.61"
+    id("org.jetbrains.kotlin.jvm") version "1.3.72"
     id("org.jlleitschuh.gradle.ktlint") version "9.1.1"
     id("org.hidetake.ssh") version "2.10.1"
     id("jacoco")

buildSrc/src/main/kotlin/Dependencies.kt

@@ -5,6 +5,7 @@ import TestVersions.JunitEngineVersion
 import TestVersions.JunitExtensionsVersion
 import TestVersions.JunitVersion
 import TestVersions.MockKVersion
+import TestVersions.RabbitmqMockVersion
 import TestVersions.XmlUnitVersion
 import Versions.CliKtVersion
 import Versions.CommonsCsvVersion
@@ -27,12 +28,13 @@ import Versions.KotlinVersion
 import Versions.Log4JVersion
 import Versions.LogbackVersion
 import Versions.MySqlVersion
+import Versions.OkHttpLoggingVersion
 import Versions.PoiVersion
-import Versions.Retrofit2CoroutineVersion
 import Versions.Retrofit2Version
 import Versions.RxJava2Version
 import Versions.ServletVersion
 import Versions.SpringAdminVersion
+import Versions.SpringBootVersion
 import Versions.SpringDataVersion
 import Versions.SpringVersion
 import Versions.SpringfoxSwaggerVersion
@@ -49,6 +51,7 @@ object TestVersions {
     const val XmlUnitVersion = "2.6.2"
     const val JsonPathAssertVersion = "2.4.0"
     const val JsonAssertVersion = "1.5.0"
+    const val RabbitmqMockVersion = "1.1.0"
 }
 
 object Versions {
@@ -57,11 +60,11 @@ object Versions {
     const val SpringVersion = "5.2.1.RELEASE"
     const val SpringAdminVersion = "2.1.6"
 
-    const val KotlinVersion = "1.3.61"
+    const val KotlinVersion = "1.3.72"
     const val KotlinLoggingVersion = "1.6.20"
     const val KotlinArrowVersion = "0.8.2"
-    const val KotlinCoroutineVersion = "1.3.3"
-    const val KMongoCoroutineVersion = "3.11.0"
+    const val KotlinCoroutineVersion = "1.3.8"
+    const val KMongoCoroutineVersion = "3.12.2"
     const val JpaEntityGraphVersion = "2.2.3"
 
     const val CommonsLang3Version = "3.8.1"
@@ -78,8 +81,8 @@ object Versions {
     const val ServletVersion = "4.0.1"
     const val HibernateEMVersion = "5.3.5.Final"
     const val JschVersion = "0.1.55"
-    const val Retrofit2Version = "2.5.0"
-    const val Retrofit2CoroutineVersion = "0.9.2"
+    const val Retrofit2Version = "2.9.0"
+    const val OkHttpLoggingVersion = "3.8.0"
     const val JSONOrgVersion = "20090211"
     const val CliKtVersion = "1.7.0"
     const val RxJava2Version = "2.2.8"
@@ -98,6 +101,7 @@ object TestDependencies {
     const val Junit5Console = "org.junit.platform:junit-platform-console:$JunitEngineVersion"
     const val Junit = "org.junit.jupiter:junit-jupiter:$JunitVersion"
     const val JunitExtensions = "io.github.glytching:junit-extensions:$JunitExtensionsVersion"
+    const val rabitMqMock = "com.github.fridujo:rabbitmq-mock:$RabbitmqMockVersion"
 
     val BaseTestCompileDependencies = arrayOf(Junit, JunitExtensions, AssertJ, MockK, KotlinTestJunit)
     val BaseTestRuntimeDependencies = arrayOf(Junit5Console)
@@ -127,7 +131,7 @@ object Dependencies {
     const val JacksonCore = "com.fasterxml.jackson.core:jackson-core:$JacksonVersion"
     const val JacksonXml = "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$JacksonVersion"
     const val Retrofit2 = "com.squareup.retrofit2:retrofit:$Retrofit2Version"
-    const val RetrofitCoroutine = "com.jakewharton.retrofit:retrofit2-kotlin-coroutines-adapter:$Retrofit2CoroutineVersion"
+    const val OkHttpLogging = "com.squareup.okhttp3:logging-interceptor:$OkHttpLoggingVersion"
 
     // Persistence
     const val HibernateEntityManager = "org.hibernate:hibernate-entitymanager:$HibernateEMVersion"
@@ -172,8 +176,10 @@ object SpringBootDependencies {
     const val SpringBootStarter = "org.springframework.boot:spring-boot-starter"
     const val SpringBootStarterTest = "org.springframework.boot:spring-boot-starter-test"
     const val SpringBootStarterWeb = "org.springframework.boot:spring-boot-starter-web"
-    const val SpringBootAmqp = "org.springframework.boot:spring-boot-starter-amqp"
+    const val SpringBootAmqp = "org.springframework.boot:spring-boot-starter-amqp:$SpringBootVersion"
+    const val SpringBootConfigurationProcessor = "org.springframework.boot:spring-boot-configuration-processor"
     const val SpringBootStarterActuator = "org.springframework.boot:spring-boot-starter-actuator"
+    const val SpringBootStarterValidation = "org.springframework.boot:spring-boot-starter-validation"
     const val SpringBootStarterDataJpa = "org.springframework.boot:spring-boot-starter-data-jpa"
     const val SpringBootStarterConfigProcessor = "org.springframework.boot:spring-boot-configuration-processor"
     const val SpringBootStarterSecurity = "org.springframework.boot:spring-boot-starter-security"

ci/submission-handlers.yml

@@ -10,6 +10,13 @@ auto-deploy-dev-handlers:
     refs:
       - dev
 
+deploy-dev-handlers:
+  stage: deploy-dev-handlers
+  when: manual
+  dependencies:
+    - build-test
+  script: gradle deployHandlers -Penv=dev -PdeployPath=/nfs/biostudies/.adm/apps/handlers/dev
+
 deploy-beta-handlers:
   stage: deploy-beta-handlers
   when: manual

client/bio-commandline/src/test/kotlin/ebi/ac/uk/paths/SubmissionFolderResolverTest.kt

@@ -1,31 +1,29 @@
 package ebi.ac.uk.paths
 
-import ebi.ac.uk.extended.model.ExtSubmission
-import io.mockk.every
-import io.mockk.impl.annotations.MockK
 import io.mockk.junit5.MockKExtension
 import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
 import java.nio.file.Paths
 
 @ExtendWith(MockKExtension::class)
-internal class SubmissionFolderResolverTest(@MockK val submission: ExtSubmission) {
-    private val testInstance = SubmissionFolderResolver(Paths.get("/tmp/nfs"))
+internal class SubmissionFolderResolverTest {
+    private val testInstance = SubmissionFolderResolver(
+        Paths.get("/tmp/nfs/submission"),
+        Paths.get("/tmp/nfs/submission/ftp")
+    )
 
     @Test
     fun getSubmissionFolder() {
         val submissionPath = "part1/part2"
-        every { submission.relPath } returns submissionPath
 
-        assertThat(testInstance.getSubmissionFolder(submission).toString())
+        assertThat(testInstance.getSubFolder(submissionPath).toString())
             .isEqualTo("/tmp/nfs/submission/part1/part2")
     }
 
     @Test
     fun getSubFilePath() {
         val submissionPath = "part1/part2"
-        every { submission.relPath } returns submissionPath
 
         assertThat(testInstance.getSubFilePath(submissionPath, "/test-file.txt").toString())
             .isEqualTo("/tmp/nfs/submission/part1/part2/Files/test-file.txt")

commons/commons-bio/src/main/kotlin/ebi/ac/uk/api/security/RegisterRequest.kt

@@ -10,7 +10,6 @@ class RegisterRequest(
     val email: String,
 
     val password: String,
-    val superUser: Boolean = false,
     var instanceKey: String? = null,
     var path: String? = null,
     val notificationsEnabled: Boolean = false,

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/AccNumber.kt

@@ -5,6 +5,6 @@ import ebi.ac.uk.base.EMPTY
 /**
  * Represents an submission accession number which include a pattern an a numeric value.
  */
-data class AccNumber(val prefix: String, val numericValue: Long? = null) {
+data class AccNumber(val prefix: String, val numericValue: String? = null) {
     override fun toString() = "$prefix${numericValue ?: EMPTY}"
 }

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/Section.kt

@@ -5,17 +5,15 @@ import ebi.ac.uk.util.collections.addLeft
 import ebi.ac.uk.util.collections.addRight
 import java.util.Objects
 
-open class Section(
+class Section(
     var type: String = "",
     var accNo: String? = null,
     var fileList: FileList? = null,
     var sections: MutableList<Either<Section, SectionsTable>> = mutableListOf(),
     var files: MutableList<Either<File, FilesTable>> = mutableListOf(),
     var links: MutableList<Either<Link, LinksTable>> = mutableListOf(),
-    override var attributes: List<Attribute> = listOf(),
-
-    // TODO Pivotal ID #168381570: Remove Parent AccNo From Section Model
-    var parentAccNo: String? = null
+    var parentAccNo: String? = null,
+    override var attributes: List<Attribute> = listOf()
 ) : Attributable {
     fun addFile(file: File) = files.addLeft(file)
     fun addLink(link: Link) = links.addLeft(link)

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/Submission.kt

@@ -1,8 +1,9 @@
 package ebi.ac.uk.model
 
 import java.util.Objects
+import java.util.Objects.equals
 
-open class Submission(
+class Submission(
     var accNo: String = "",
     var section: Section = Section(),
     var tags: MutableList<Pair<String, String>> = mutableListOf(),
@@ -11,10 +12,12 @@ open class Submission(
     override fun equals(other: Any?) = when {
         other !is Submission -> false
         other === this -> true
-        else -> Objects.equals(accNo, other.accNo)
-            .and(Objects.equals(section, other.section))
-            .and(Objects.equals(attributes, other.attributes))
+        else -> equals(accNo, other.accNo)
+            .and(equals(section, other.section))
+            .and(equals(attributes, other.attributes))
     }
 
     override fun hashCode() = Objects.hash(accNo, section, attributes)
+
+    override fun toString() = "Submission(accNo='$accNo',attributes=$attributes)"
 }

commons/commons-bio/src/main/kotlin/ebi/ac/uk/model/TableModel.kt

@@ -91,6 +91,5 @@ class SectionsTable(sections: List<Section> = emptyList()) : Table<Section>(sect
         override val attributes = t.attributes
     }
 
-    fun asSectionsTable() = SectionsTable(
-        elements.map { if (it is ExtendedSection) it.asSection() else it })
+    fun asSectionsTable() = SectionsTable(elements.map { it })
 }