Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pivotal ID # 185649200: FIRE Duplicated Folder Name #748

Merged
merged 3 commits into from
Sep 28, 2023
Merged

Pivotal ID # 185649200: FIRE Duplicated Folder Name #748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
47410c6
Pivotal ID # 185649200: FIRE Duplicated Folder Name
jhoanmanuelms Sep 19, 2023
14f954b
Merge branch 'master' into bugfix/pivotal-#185756018-duplicated-direc…
jhoanmanuelms Sep 27, 2023
f32fe9f
Pivotal ID # 185649200: FIRE Duplicated Folder Name
jhoanmanuelms Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 100 additions & 33 deletions ...rc/itest/kotlin/ac/uk/ebi/biostd/itest/test/submission/submit/SubmissionFileSourceTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ class SubmissionFileSourceTest(
fun `6-3-1 submission with directory with files on FIRE`() = runTest {
val submission = tsv {
line("Submission", "S-FSTST3")
line("Title", "Simple Submission With directory")
line("Title", "Simple Submission With directory on FIRE")
line()

line("Study")
Expand All @@ -218,26 +218,18 @@ class SubmissionFileSourceTest(
line("File", "directory")
line("Type", "test")
line()

line("File", "folder/inner")
line("Type", "inner folder")
line()
}.toString()

val file1 = tempFolder.createFile("file1.txt", "content-1")
val file2 = tempFolder.createFile(".file2.txt", "content-2")
val file3 = tempFolder.createFile("file3.txt", "content-3")
val file4 = tempFolder.createFile("file4.txt", "content-4")

webClient.uploadFiles(listOf(file1), "directory")
webClient.uploadFiles(listOf(file2), "directory/subdirectory")
webClient.uploadFiles(listOf(file3), "folder")
webClient.uploadFiles(listOf(file4), "folder/inner")

assertThat(webClient.submitSingle(submission, TSV)).isSuccessful()

val submitted = submissionRepository.getExtByAccNo("S-FSTST3")
assertThat(submitted.section.files).hasSize(2)
assertThat(submitted.section.files).hasSize(1)
assertThat(submitted.section.files.first()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(328L)
Expand All @@ -249,14 +241,6 @@ class SubmissionFileSourceTest(
"subdirectory/.file2.txt" to file2.readText()
)
}
assertThat(submitted.section.files.second()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(161L)
assertThat(it.md5).isEqualTo("1FBD0EEC5299D755DF21D2DA7567EAA3")

val files = getZipFiles("$submissionPath/${submitted.relPath}/Files/folder/inner.zip")
assertThat(files).containsExactly("file4.txt" to file4.readText())
}
}

@Test
Expand Down Expand Up @@ -363,12 +347,62 @@ class SubmissionFileSourceTest(
}
}

@Test
@EnabledIfSystemProperty(named = "enableFire", matches = "true")
fun `6-3-4 submission with directories with the same name on FIRE`() = runTest {
val submission = tsv {
line("Submission", "S-FSTST34")
line("Title", "Directories With The Same Name on FIRE")
line()

line("Study")
line()

line("File", "duplicated")
line("Type", "A")
line()

line("File", "folder/duplicated")
line("Type", "B")
line()
}.toString()

val fileA = tempFolder.createFile("fileA.txt", "one content")
val fileB = tempFolder.createFile("fileB.txt", "another content")
val fileC = tempFolder.createFile("fileC.txt", "yet another content")

webClient.uploadFiles(listOf(fileA), "duplicated")
webClient.uploadFiles(listOf(fileC), "folder")
webClient.uploadFiles(listOf(fileB), "folder/duplicated")

assertThat(webClient.submitSingle(submission, TSV)).isSuccessful()

val submitted = submissionRepository.getExtByAccNo("S-FSTST34")
assertThat(submitted.section.files).hasSize(2)
assertThat(submitted.section.files.first()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(163L)
assertThat(it.md5).isEqualTo("EEB90F918DF18A5DA2F5C7626900083B")

val files = getZipFiles("$submissionPath/${submitted.relPath}/Files/duplicated.zip")
assertThat(files).containsExactly("fileA.txt" to fileA.readText())
}
assertThat(submitted.section.files.second()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(167L)
assertThat(it.md5).isEqualTo("414A43404B81150677559D28C0DB9F4B")

val files = getZipFiles("$submissionPath/${submitted.relPath}/Files/folder/duplicated.zip")
assertThat(files).containsExactly("fileB.txt" to fileB.readText())
}
}

@Test
@EnabledIfSystemProperty(named = "enableFire", matches = "false")
fun `6-3-4 submission with directory with files on NFS`() = runTest {
fun `6-3-5 submission with directory with files on NFS`() = runTest {
val submission = tsv {
line("Submission", "S-FSTST4")
line("Title", "Simple Submission With directory")
line("Title", "Simple Submission With directory on NFS")
line()

line("Study")
Expand All @@ -377,26 +411,18 @@ class SubmissionFileSourceTest(
line("File", "directory")
line("Type", "test")
line()

line("File", "folder/inner")
line("Type", "inner folder")
line()
}.toString()

val file1 = tempFolder.createFile("file1.txt", "content-1")
val file2 = tempFolder.createFile(".file2.txt", "content-2")
val file3 = tempFolder.createFile("file3.txt", "content-3")
val file4 = tempFolder.createFile("file4.txt", "content-4")

webClient.uploadFiles(listOf(file1), "directory")
webClient.uploadFiles(listOf(file2), "directory/subdirectory")
webClient.uploadFiles(listOf(file3), "folder")
webClient.uploadFiles(listOf(file4), "folder/inner")

assertThat(webClient.submitSingle(submission, TSV)).isSuccessful()

val submitted = submissionRepository.getExtByAccNo("S-FSTST4")
assertThat(submitted.section.files).hasSize(2)
assertThat(submitted.section.files).hasSize(1)
assertThat(submitted.section.files.first()).hasLeftValueSatisfying {

assertThat(it.type).isEqualTo(ExtFileType.DIR)
Expand All @@ -407,14 +433,55 @@ class SubmissionFileSourceTest(
assertDirFile(dir.resolve("file1.txt"), "content-1")
assertDirFile(dir.resolve("subdirectory/.file2.txt"), "content-2")
}
}

@Test
@EnabledIfSystemProperty(named = "enableFire", matches = "false")
fun `6-3-6 submission with directories with the same name on NFS`() = runTest {
val submission = tsv {
line("Submission", "S-FSTST34")
line("Title", "Directories With The Same Name on NFS")
line()

line("Study")
line()

line("File", "duplicated")
line("Type", "A")
line()

line("File", "folder/duplicated")
line("Type", "B")
line()
}.toString()

val fileA = tempFolder.createFile("fileA.txt", "one content")
val fileB = tempFolder.createFile("fileB.txt", "another content")
val fileC = tempFolder.createFile("fileC.txt", "yet another content")

webClient.uploadFiles(listOf(fileA), "duplicated")
webClient.uploadFiles(listOf(fileC), "folder")
webClient.uploadFiles(listOf(fileB), "folder/duplicated")

assertThat(webClient.submitSingle(submission, TSV)).isSuccessful()

val submitted = submissionRepository.getExtByAccNo("S-FSTST34")
assertThat(submitted.section.files).hasSize(2)
assertThat(submitted.section.files.first()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(11L)
assertThat(it.md5).isEmpty()

val dir = Paths.get("$submissionPath/${submitted.relPath}/Files/duplicated")
assertDirFile(dir.resolve("fileA.txt"), "one content")
}
assertThat(submitted.section.files.second()).hasLeftValueSatisfying {
assertThat(it.type).isEqualTo(ExtFileType.DIR)
assertThat(it.size).isEqualTo(9L)
assertThat(it.size).isEqualTo(15L)
assertThat(it.md5).isEmpty()

val dir = Paths.get("$submissionPath/${submitted.relPath}/Files/folder")
assertThat(dir.resolve("file3.txt").exist()).isFalse()
assertDirFile(dir.resolve("inner/file4.txt"), "content-4")
val dir = Paths.get("$submissionPath/${submitted.relPath}/Files/folder/duplicated")
assertDirFile(dir.resolve("fileB.txt"), "another content")
}
}

Expand Down
4 changes: 3 additions & 1 deletion ...ission/submission-webapp/src/itest/resources/itestsInventory/itestsInventory.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@
| SubmissionFileSourceTest | 6-3-1 | submission with directory with files on FIRE | |
| SubmissionFileSourceTest | 6-3-2 | re-submission with directory with files on FIRE | |
| SubmissionFileSourceTest | 6-3-3 | re-submission on FIRE, User folder should be prioritized | |
| SubmissionFileSourceTest | 6-3-4 | submission with directory with files on NFS | |
| SubmissionFileSourceTest | 6-3-4 | submission with directories with the same name on FIRE | |
| SubmissionFileSourceTest | 6-3-5 | submission with directory with files on NFS | |
| SubmissionFileSourceTest | 6-3-6 | submission with directories with the same name on NFS | |
| SubmissionFileSourceTest | 6-4 | multiple file references | |
| SubmissionFileSourceTest | 6-5 | submission with group file | |
| SubmissionFileSourceTest | 6-6 | Submission bypassing fire | |
Expand Down
6 changes: 3 additions & 3 deletions .../src/main/kotlin/ac/uk/ebi/biostd/submission/submitter/request/SubmissionRequestLoader.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import ebi.ac.uk.extended.model.ExtSubmission
import ebi.ac.uk.extended.model.FireFile
import ebi.ac.uk.extended.model.NfsFile
import ebi.ac.uk.extended.model.StorageMode.FIRE
import ebi.ac.uk.io.ext.createTempFile
import ebi.ac.uk.io.ext.md5
import ebi.ac.uk.io.ext.size
import kotlinx.coroutines.async
Expand All @@ -19,7 +20,6 @@ import kotlinx.coroutines.flow.map
import kotlinx.coroutines.supervisorScope
import mu.KotlinLogging
import java.io.File
import java.nio.file.Files

private val logger = KotlinLogging.logger {}

Expand Down Expand Up @@ -76,9 +76,9 @@ class SubmissionRequestLoader(
val tempFolder = fireTempDirPath.resolve("$accNo/$version")
tempFolder.mkdirs()

val target = tempFolder.resolve("${file.name}.zip")
Files.deleteIfExists(target.toPath())
val target = tempFolder.createTempFile(directory.fileName, ".zip")
ZipUtil.pack(file, target)

return target
}

Expand Down