Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pivotal ID # 183440873: Storage Mode Option Ignored By CLI #621

Merged
merged 2 commits into from
Oct 4, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.options.required
import com.github.ajalt.clikt.parameters.types.file
import ebi.ac.uk.extended.model.StorageMode
import ebi.ac.uk.extended.model.StorageMode.FIRE
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.ON_BEHALF_HELP
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.PASSWORD_HELP
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.SERVER_HELP
Expand Down Expand Up @@ -33,11 +34,11 @@ internal class SubmitAsyncCommand(
private val storageMode by option("-sm", "--storageMode", help = STORAGE_MODE)

override fun run() {
val mode = storageMode?.let { StorageMode.fromString(it) }
val mode = storageMode?.let { StorageMode.fromString(it) } ?: FIRE
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can use default option when register parameter

val securityConfig = SecurityConfig(server, user, password, onBehalf)
val filesConfig = SubmissionFilesConfig(splitFiles(attached), splitPreferredSources(preferredSources))
val filesConfig = SubmissionFilesConfig(splitFiles(attached), mode, splitPreferredSources(preferredSources))

submissionService.submitAsync(SubmissionRequest(input, mode, securityConfig, filesConfig))
submissionService.submitAsync(SubmissionRequest(input, securityConfig, filesConfig))
echo("SUCCESS: Submission is in queue to be submitted")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.options.required
import com.github.ajalt.clikt.parameters.types.file
import ebi.ac.uk.extended.model.StorageMode
import ebi.ac.uk.extended.model.StorageMode.FIRE
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.ON_BEHALF_HELP
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.PASSWORD_HELP
import uk.ac.ebi.biostd.client.cli.common.CommonParameters.SERVER_HELP
Expand Down Expand Up @@ -33,11 +34,11 @@ internal class SubmitCommand(
private val storageMode by option("-sm", "--storageMode", help = STORAGE_MODE)

override fun run() {
val mode = storageMode?.let { StorageMode.fromString(it) }
val mode = storageMode?.let { StorageMode.fromString(it) } ?: FIRE
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above

val securityConfig = SecurityConfig(server, user, password, onBehalf)
val filesConfig = SubmissionFilesConfig(splitFiles(attached), splitPreferredSources(preferredSources))
val filesConfig = SubmissionFilesConfig(splitFiles(attached), mode, splitPreferredSources(preferredSources))

val response = subService.submit(SubmissionRequest(input, mode, securityConfig, filesConfig))
val response = subService.submit(SubmissionRequest(input, securityConfig, filesConfig))
echo("SUCCESS: Submission with AccNo ${response.accNo} was submitted")
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package uk.ac.ebi.biostd.client.cli.dto

import ac.uk.ebi.biostd.client.integration.web.SubmissionFilesConfig
import ebi.ac.uk.extended.model.StorageMode
import java.io.File

internal data class SubmissionRequest(
val submissionFile: File,
val storageMode: StorageMode?,
val securityConfig: SecurityConfig,
val filesConfig: SubmissionFilesConfig,
)
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import uk.ac.ebi.biostd.client.cli.dto.ValidateFileListRequest
internal class SubmissionService {
fun submit(rqt: SubmissionRequest): Submission = performRequest {
val client = bioWebClient(rqt.securityConfig)
return client.submitSingle(rqt.submissionFile, rqt.storageMode, rqt.filesConfig).body
return client.submitSingle(rqt.submissionFile, rqt.filesConfig).body
}

fun submitAsync(request: SubmissionRequest) = performRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import ac.uk.ebi.biostd.client.integration.web.SubmissionFilesConfig
import com.github.ajalt.clikt.core.IncorrectOptionValueCount
import com.github.ajalt.clikt.core.MissingParameter
import ebi.ac.uk.extended.model.StorageMode
import ebi.ac.uk.io.sources.PreferredSource.FIRE
import ebi.ac.uk.io.sources.PreferredSource.SUBMISSION
import ebi.ac.uk.io.sources.PreferredSource
import ebi.ac.uk.test.clean
import io.github.glytching.junit.extension.folder.TemporaryFolder
import io.github.glytching.junit.extension.folder.TemporaryFolderExtension
Expand Down Expand Up @@ -44,8 +43,8 @@ internal class SubmitAsyncCommandTest(
val attachedFile2 = temporaryFolder.createFile("attachedFile2.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), emptyList())
val request = SubmissionRequest(submission, null, securityConfig, filesConfig)
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), StorageMode.FIRE, emptyList())
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submitAsync(request) } answers { nothing }

Expand All @@ -69,8 +68,9 @@ internal class SubmitAsyncCommandTest(
val attachedFile2 = temporaryFolder.createFile("attachedFile2.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), listOf(SUBMISSION, FIRE))
val request = SubmissionRequest(submission, StorageMode.NFS, securityConfig, filesConfig)
val sources = listOf(PreferredSource.SUBMISSION, PreferredSource.FIRE)
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), StorageMode.NFS, sources)
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submitAsync(request) } answers { nothing }

Expand All @@ -94,8 +94,8 @@ internal class SubmitAsyncCommandTest(
val submission = temporaryFolder.createFile("Submission.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(emptyList(), emptyList())
val request = SubmissionRequest(submission, StorageMode.FIRE, securityConfig, filesConfig)
val filesConfig = SubmissionFilesConfig(emptyList(), StorageMode.FIRE, emptyList())
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submitAsync(request) } answers { nothing }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@ import ac.uk.ebi.biostd.client.integration.web.SubmissionFilesConfig
import com.github.ajalt.clikt.core.IncorrectOptionValueCount
import com.github.ajalt.clikt.core.MissingParameter
import ebi.ac.uk.extended.model.StorageMode
import ebi.ac.uk.io.sources.PreferredSource.FIRE
import ebi.ac.uk.io.sources.PreferredSource.SUBMISSION
import ebi.ac.uk.io.sources.PreferredSource
import ebi.ac.uk.model.Submission
import ebi.ac.uk.test.clean
import io.github.glytching.junit.extension.folder.TemporaryFolder
Expand Down Expand Up @@ -47,8 +46,8 @@ internal class SubmitCommandTest(
val attachedFile2 = temporaryFolder.createFile("attachedFile2.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), emptyList())
val request = SubmissionRequest(submission, null, securityConfig, filesConfig)
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), StorageMode.FIRE, emptyList())
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submit(request) } returns mockResponse

Expand All @@ -74,8 +73,9 @@ internal class SubmitCommandTest(
val attachedFile2 = temporaryFolder.createFile("attachedFile2.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), listOf(SUBMISSION, FIRE))
val request = SubmissionRequest(submission, StorageMode.NFS, securityConfig, filesConfig)
val sources = listOf(PreferredSource.SUBMISSION, PreferredSource.FIRE)
val filesConfig = SubmissionFilesConfig(listOf(attachedFile1, attachedFile2), StorageMode.NFS, sources)
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submit(request) } returns mockResponse

Expand All @@ -101,8 +101,8 @@ internal class SubmitCommandTest(
val submission = temporaryFolder.createFile("Submission.tsv")

val securityConfig = SecurityConfig("server", "user", "password")
val filesConfig = SubmissionFilesConfig(emptyList(), emptyList())
val request = SubmissionRequest(submission, StorageMode.FIRE, securityConfig, filesConfig)
val filesConfig = SubmissionFilesConfig(emptyList(), StorageMode.FIRE, emptyList())
val request = SubmissionRequest(submission, securityConfig, filesConfig)

every { submissionService.submit(request) } returns mockResponse

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ internal class SubmissionServiceTest {
fun submit() {
every { create(SERVER).getAuthenticatedClient(USER, PASSWORD, ON_BEHALF) } returns bioWebClient
every {
bioWebClient.submitSingle(subRequest.submissionFile, FIRE, subRequest.filesConfig).body
bioWebClient.submitSingle(subRequest.submissionFile, subRequest.filesConfig).body
} returns submission

val submitted = testInstance.submit(subRequest)

assertThat(submitted).isEqualTo(submission)
verify(exactly = 1) {
create(SERVER).getAuthenticatedClient(USER, PASSWORD, ON_BEHALF)
bioWebClient.submitSingle(subRequest.submissionFile, FIRE, subRequest.filesConfig)
bioWebClient.submitSingle(subRequest.submissionFile, subRequest.filesConfig)
}
}

Expand Down Expand Up @@ -151,9 +151,9 @@ internal class SubmissionServiceTest {
private val submission: Submission = mockk()
private val bioWebClient: BioWebClient = mockk()
private val securityConfig = SecurityConfig(SERVER, USER, PASSWORD, ON_BEHALF)
private val filesConfig = SubmissionFilesConfig(listOf(mockk()), listOf(SUBMISSION))
private val filesConfig = SubmissionFilesConfig(listOf(mockk()), FIRE, listOf(SUBMISSION))

private val subRequest = SubmissionRequest(mockk(), FIRE, securityConfig, filesConfig)
private val subRequest = SubmissionRequest(mockk(), securityConfig, filesConfig)
private val deletionRequest = DeletionRequest(securityConfig, accNoList = listOf(ACC_NO))
private val validateFileList = ValidateFileListRequest(FILE_LIST_PATH, ROOT_PATH, ACC_NO, securityConfig)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import ac.uk.ebi.biostd.integration.SubFormat.Companion.JSON
import ac.uk.ebi.biostd.integration.SubFormat.JsonFormat.JsonPretty
import ebi.ac.uk.api.ClientResponse
import ebi.ac.uk.extended.model.ExtAttributeDetail
import ebi.ac.uk.extended.model.StorageMode
import ebi.ac.uk.model.Submission
import ebi.ac.uk.model.constants.ATTRIBUTES
import org.springframework.core.io.FileSystemResource
Expand All @@ -34,7 +33,6 @@ internal class MultiPartSubmitClient(
) : MultipartSubmitOperations {
override fun submitSingle(
submission: File,
storageMode: StorageMode?,
filesConfig: SubmissionFilesConfig,
attrs: Map<String, String>,
): SubmissionResponse {
Expand All @@ -50,7 +48,6 @@ internal class MultiPartSubmitClient(
override fun submitSingle(
submission: String,
format: SubmissionFormat,
storageMode: StorageMode?,
filesConfig: SubmissionFilesConfig,
): SubmissionResponse {
val headers = createHeaders(format)
Expand All @@ -62,7 +59,6 @@ internal class MultiPartSubmitClient(
override fun submitSingle(
submission: Submission,
format: SubmissionFormat,
storageMode: StorageMode?,
filesConfig: SubmissionFilesConfig,
): SubmissionResponse {
val headers = createHeaders(format)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ac.uk.ebi.biostd.client.common

import ac.uk.ebi.biostd.client.integration.web.SubmissionFilesConfig
import ebi.ac.uk.api.STORAGE_MODE
import ebi.ac.uk.model.constants.FILES
import ebi.ac.uk.model.constants.PREFERRED_SOURCES
import ebi.ac.uk.model.constants.SUBMISSION
Expand All @@ -9,13 +10,14 @@ import org.springframework.util.LinkedMultiValueMap

internal fun getMultipartBody(
filesConfig: SubmissionFilesConfig,
submission: Any
submission: Any,
): LinkedMultiValueMap<String, Any> {
val (files, preferredSources) = filesConfig
val (files, storageMode, preferredSources) = filesConfig

return LinkedMultiValueMap(
files.map { FILES to FileSystemResource(it) }
.plus(preferredSources.map { PREFERRED_SOURCES to it.name })
.plus(STORAGE_MODE to storageMode.value)
.plus(SUBMISSION to submission)
.groupBy({ it.first }, { it.second })
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ typealias SubmissionResponse = ClientResponse<Submission>

data class SubmissionFilesConfig(
val files: List<File>,
val storageMode: StorageMode,
val preferredSources: List<PreferredSource> = emptyList(),
)

Expand Down Expand Up @@ -135,20 +136,17 @@ interface MultipartSubmitOperations {
fun submitSingle(
submission: String,
format: SubmissionFormat,
storageMode: StorageMode? = null,
filesConfig: SubmissionFilesConfig,
): SubmissionResponse

fun submitSingle(
submission: Submission,
format: SubmissionFormat,
storageMode: StorageMode? = null,
filesConfig: SubmissionFilesConfig,
): SubmissionResponse

fun submitSingle(
submission: File,
storageMode: StorageMode? = null,
filesConfig: SubmissionFilesConfig,
attrs: Map<String, String> = emptyMap(),
): SubmissionResponse
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ class PmcSubmitter(
runCatching {
logger.info { "submitting accNo='${submission.accNo}'" }
val files = submissionService.getSubFiles(submission.files).map { File(it.path) }
val filesConfig = SubmissionFilesConfig(files)
bioWebClient.submitSingle(submission.body, SubmissionFormat.JSON, StorageMode.NFS, filesConfig)
val filesConfig = SubmissionFilesConfig(files, StorageMode.NFS)
bioWebClient.submitSingle(submission.body, SubmissionFormat.JSON, filesConfig)
}.fold(
{ submissionService.changeStatus(submission, SubmissionStatus.SUBMITTED) },
{ errorDocService.saveError(submission, PmcMode.SUBMIT, it) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ class DeletePermissionTest(
}.toString()
val projectFile = tempFolder.createFile("a-collection.tsv", project)

val filesConfig = SubmissionFilesConfig(emptyList())
assertThat(superUserWebClient.submitSingle(projectFile, storageMode, filesConfig)).isSuccessful()
val filesConfig = SubmissionFilesConfig(emptyList(), storageMode)
assertThat(superUserWebClient.submitSingle(projectFile, filesConfig)).isSuccessful()
}

private fun setUpTestUsers() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ class SubmissionListApiTest(
assertThat(webClient.submitSingle(getSimpleSubmission(idx), TSV)).isSuccessful()
}

val filesConfig = SubmissionFilesConfig(emptyList())
val filesConfig = SubmissionFilesConfig(emptyList(), storageMode)
for (idx in 21..30) {
val submission = tempFolder.createFile("submission$idx.tsv", getSimpleSubmission(idx))
assertThat(webClient.submitSingle(submission, storageMode, filesConfig)).isSuccessful()
assertThat(webClient.submitSingle(submission, filesConfig)).isSuccessful()
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ class FileListSubmissionTest(
}.toString()
)

val filesConfig = SubmissionFilesConfig(listOf(fileList, tempFolder.createFile("File4.txt")))
val response = webClient.submitSingle(submission, JSON, storageMode, filesConfig)
val filesConfig = SubmissionFilesConfig(listOf(fileList, tempFolder.createFile("File4.txt")), storageMode)
val response = webClient.submitSingle(submission, JSON, filesConfig)

assertThat(response).isSuccessful()
assertSubmissionFiles("S-TEST4", "File4.txt", "FileList")
Expand Down Expand Up @@ -138,8 +138,8 @@ class FileListSubmissionTest(
}
}

val filesConfig = SubmissionFilesConfig(listOf(fileList, tempFolder.createFile("File5.txt")))
val response = webClient.submitSingle(submission, JSON, storageMode, filesConfig)
val filesConfig = SubmissionFilesConfig(listOf(fileList, tempFolder.createFile("File5.txt")), storageMode)
val response = webClient.submitSingle(submission, JSON, filesConfig)

assertThat(response).isSuccessful()
assertSubmissionFiles("S-TEST5", "File5.txt", "FileList")
Expand Down Expand Up @@ -171,9 +171,9 @@ class FileListSubmissionTest(
}
}.toString()

val filesConfig = SubmissionFilesConfig(listOf(fileList))
val filesConfig = SubmissionFilesConfig(listOf(fileList), storageMode)
assertThatExceptionOfType(WebClientException::class.java)
.isThrownBy { webClient.submitSingle(submission, JSON, storageMode, filesConfig) }
.isThrownBy { webClient.submitSingle(submission, JSON, filesConfig) }
.withMessageContaining("Unsupported page tab format FileList.txt")
}

Expand All @@ -199,8 +199,8 @@ class FileListSubmissionTest(
)

webClient.uploadFile(fileList, "folder")
val filesConfig = SubmissionFilesConfig(listOf(referencedFile))
assertThat(webClient.submitSingle(submission, TSV, storageMode, filesConfig)).isSuccessful()
val filesConfig = SubmissionFilesConfig(listOf(referencedFile), storageMode)
assertThat(webClient.submitSingle(submission, TSV, filesConfig)).isSuccessful()

val extSubmission = webClient.getExtByAccNo("S-TEST6")
val referencedFiles = webClient.getReferencedFiles(extSubmission.section.fileList!!.filesUrl!!).files
Expand Down Expand Up @@ -235,8 +235,8 @@ class FileListSubmissionTest(
)

val firstVersion = submission("reusable-file-list.tsv")
val filesConfig = SubmissionFilesConfig(listOf(fileList, referencedFile))
assertThat(webClient.submitSingle(firstVersion, TSV, storageMode, filesConfig)).isSuccessful()
val filesConfig = SubmissionFilesConfig(listOf(fileList, referencedFile), storageMode)
assertThat(webClient.submitSingle(firstVersion, TSV, filesConfig)).isSuccessful()
assertSubmissionFiles("S-TEST7", "File7.txt", "reusable-file-list")
fileList.delete()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ class FileListValidationTest(
val fileList = tempFolder.createFile("valid-file-list.tsv", fileListContent)

webClient.uploadFiles(listOf(file1, fileList))
webClient.submitSingle(previousVersion, TSV, storageMode, SubmissionFilesConfig(listOf(file2)))
webClient.submitSingle(previousVersion, TSV, SubmissionFilesConfig(listOf(file2), storageMode))

webClient.validateFileList(fileList.name, accNo = "S-FLV123")

Expand Down Expand Up @@ -149,7 +149,7 @@ class FileListValidationTest(

fireClient.save(file2, file2Md5, file2.size())
webClient.uploadFiles(listOf(file1, fileList))
webClient.submitSingle(previousVersion, TSV, storageMode, SubmissionFilesConfig(listOf(file3)))
webClient.submitSingle(previousVersion, TSV, SubmissionFilesConfig(listOf(file3), storageMode))

webClient.validateFileList(fileList.name, accNo = "S-FLV124")

Expand Down
Loading