test: remove tsv tests from matrix e2e tests and rename ndjson and ts…

…v directories
GenSpectrum · Oct 7, 2024 · c1cb660 · c1cb660
1 parent 8dc15ea
commit c1cb660
Show file tree

Hide file tree

Showing 35 changed files with 48 additions and 29 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -163,7 +163,6 @@ jobs:
     strategy:
       matrix:
         preprocessing-docker-compose: [
-          docker-compose-for-tests-preprocessing-from-tsv.yml,
           docker-compose-for-tests-preprocessing-from-ndjson.yml
         ]
     steps:

diff --git a/docker-compose-for-tests-preprocessing-from-ndjson.yml b/docker-compose-for-tests-preprocessing-from-ndjson.yml
@@ -3,10 +3,10 @@ services:
   silo:
     image: ${SILO_IMAGE}
     volumes:
-      - ./testBaseData/exampleDatasetAsNdjson:/preprocessing/input
+      - ./testBaseData/exampleDataset:/preprocessing/input
       - ./testBaseData/output:/preprocessing/output
-      - ./testBaseData/exampleDatasetAsNdjson/preprocessing_config.yaml:/app/preprocessing_config.yaml
-      - ./testBaseData/exampleDatasetAsNdjson/database_config.yaml:/app/database_config.yaml
+      - ./testBaseData/exampleDataset/preprocessing_config.yaml:/app/preprocessing_config.yaml
+      - ./testBaseData/exampleDataset/database_config.yaml:/app/database_config.yaml
       - ./logs:/app/logs
     command:
       - "--preprocessing"
diff --git a/docker-compose-for-tests-preprocessing-from-tsv.yml b/docker-compose-for-tests-preprocessing-from-tsv.yml
diff --git a/src/silo/config/util/yaml_file.test.cpp b/src/silo/config/util/yaml_file.test.cpp
@@ -24,13 +24,13 @@ TEST(YamlFile, canCorrectlyCheckForNonPresentProperties) {
 TEST(YamlFile, getStringGetsCorrectField) {
    const YamlFile under_test("./testBaseData/test_preprocessing_config.yaml");
 
-   ASSERT_EQ(under_test.getString({{"inputDirectory"}}), "./testBaseData/exampleDataset/");
+   ASSERT_EQ(under_test.getString({{"inputDirectory"}}), "./testBaseData/exampleDatasetAsTsv/");
 }
 
 TEST(YamlFile, getStringGetsCorrectFieldsRepeatedly) {
    const YamlFile under_test("./testBaseData/test_preprocessing_config.yaml");
 
-   ASSERT_EQ(under_test.getString({{"inputDirectory"}}), "./testBaseData/exampleDataset/");
+   ASSERT_EQ(under_test.getString({{"inputDirectory"}}), "./testBaseData/exampleDatasetAsTsv/");
    ASSERT_EQ(under_test.getString({{"outputDirectory"}}), "./output/");
    ASSERT_EQ(under_test.getString({{"metadataFilename"}}), "small_metadata_set.tsv");
    ASSERT_EQ(under_test.getString({{"lineageDefinitionsFilename"}}), "lineage_definitions.yaml");
@@ -49,7 +49,7 @@ TEST(YamlFile, shouldReadConfigWithCorrectParametersAndDefaults) {
 
    ASSERT_NO_THROW(config.overwrite(YamlFile("./testBaseData/test_preprocessing_config.yaml")););
 
-   const std::string input_directory = "./testBaseData/exampleDataset/";
+   const std::string input_directory = "./testBaseData/exampleDatasetAsTsv/";
    ASSERT_EQ(config.getMetadataInputFilename(), input_directory + "small_metadata_set.tsv");
    ASSERT_EQ(config.getLineageDefinitionsFilename(), input_directory + "lineage_definitions.yaml");
 }
@@ -69,7 +69,7 @@ TEST(YamlFile, shouldReadConfigWithOverriddenDefaults) {
       YamlFile("./testBaseData/test_preprocessing_config_with_overridden_defaults.yaml")
    ););
 
-   const std::string input_directory = "./testBaseData/exampleDataset/";
+   const std::string input_directory = "./testBaseData/exampleDatasetAsTsv/";
    ASSERT_EQ(config.getMetadataInputFilename(), input_directory + "small_metadata_set.tsv");
    ASSERT_EQ(config.getLineageDefinitionsFilename(), input_directory + "lineage_definitions.yaml");
 

diff --git a/src/silo/preprocessing/metadata_info.test.cpp b/src/silo/preprocessing/metadata_info.test.cpp
@@ -29,7 +29,7 @@ TEST(
 
    EXPECT_THROW(
       silo::preprocessing::MetadataInfo::validateMetadataFile(
-         "testBaseData/exampleDataset/small_metadata_set.tsv",
+         "testBaseData/exampleDatasetAsTsv/small_metadata_set.tsv",
          some_config_with_one_column_not_in_metadata
       ),
       silo::preprocessing::PreprocessingException

diff --git a/src/silo/preprocessing/sequence_info.test.cpp b/src/silo/preprocessing/sequence_info.test.cpp
@@ -19,7 +19,7 @@ TEST(SequenceInfo, validatesSuccessfulOnCorrectFile) {
 
 TEST(SequenceInfo, failWhenTooManyGenomesInReferences) {
    const auto reference_genomes =
-      ReferenceGenomes::readFromFile("testBaseData/exampleDataset/reference_genomes.json");
+      ReferenceGenomes::readFromFile("testBaseData/exampleDatasetAsTsv/reference_genomes.json");
 
    ASSERT_THROW(
       SequenceInfo::validateNdjsonFile(

diff --git a/src/silo/storage/reference_genomes.test.cpp b/src/silo/storage/reference_genomes.test.cpp
@@ -7,7 +7,8 @@
 
 TEST(ReferenceGenome, readFromFile) {
    auto under_test =
-      silo::ReferenceGenomes::readFromFile("testBaseData/exampleDataset/reference_genomes.json");
+      silo::ReferenceGenomes::readFromFile("testBaseData/exampleDatasetAsTsv/reference_genomes.json"
+      );
 
    ASSERT_EQ(under_test.nucleotide_sequences.size(), 2UL);
    ASSERT_EQ(under_test.aa_sequences.size(), 12UL);

diff --git a/.../exampleDatasetAsNdjson/input_file.ndjson → ...BaseData/exampleDataset/input_file.ndjson b/.../exampleDatasetAsNdjson/input_file.ndjson → ...BaseData/exampleDataset/input_file.ndjson
diff --git a/testBaseData/exampleDataset/preprocessing_config.yaml b/testBaseData/exampleDataset/preprocessing_config.yaml
@@ -1,3 +1,4 @@
-metadataFilename: "small_metadata_set.tsv"
+ndjsonInputFilename: "input_file.ndjson"
 lineageDefinitionsFilename: "lineage_definitions.yaml"
 referenceGenomeFilename: "reference_genomes.json"
+duckdbMemoryLimitInG: 20
diff --git a/testBaseData/exampleDatasetAsNdjson/database_config.yaml b/testBaseData/exampleDatasetAsNdjson/database_config.yaml
diff --git a/...BaseData/exampleDataset/aa_insertions.tsv → ...ata/exampleDatasetAsTsv/aa_insertions.tsv b/...BaseData/exampleDataset/aa_insertions.tsv → ...ata/exampleDatasetAsTsv/aa_insertions.tsv
diff --git a/testBaseData/exampleDatasetAsTsv/database_config.yaml b/testBaseData/exampleDatasetAsTsv/database_config.yaml
@@ -0,0 +1,32 @@
+schema:
+  instanceName: sars_cov-2_minimal_test_config
+  metadata:
+    - name: gisaid_epi_isl
+      type: string
+    - name: date
+      type: date
+    - name: unsorted_date
+      type: date
+    - name: region
+      type: string
+      generateIndex: true
+    - name: country
+      type: string
+      generateIndex: true
+    - name: pango_lineage
+      type: string
+      generateIndex: true
+      lineageIndex: true
+    - name: division
+      type: string
+      generateIndex: true
+    - name: age
+      type: int
+    - name: qc_value
+      type: float
+    - name: test_boolean_column
+      type: boolean
+  primaryKey: gisaid_epi_isl
+  dateToSortBy: date
+  partitionBy: pango_lineage
+defaultNucleotideSequence: "main"
diff --git a/testBaseData/exampleDataset/gene_0.fasta → ...BaseData/exampleDatasetAsTsv/gene_0.fasta b/testBaseData/exampleDataset/gene_0.fasta → ...BaseData/exampleDatasetAsTsv/gene_0.fasta
diff --git a/testBaseData/exampleDataset/gene_1.fasta → ...BaseData/exampleDatasetAsTsv/gene_1.fasta b/testBaseData/exampleDataset/gene_1.fasta → ...BaseData/exampleDatasetAsTsv/gene_1.fasta
diff --git a/testBaseData/exampleDataset/gene_10.fasta → ...aseData/exampleDatasetAsTsv/gene_10.fasta b/testBaseData/exampleDataset/gene_10.fasta → ...aseData/exampleDatasetAsTsv/gene_10.fasta
diff --git a/testBaseData/exampleDataset/gene_11.fasta.xz → ...Data/exampleDatasetAsTsv/gene_11.fasta.xz b/testBaseData/exampleDataset/gene_11.fasta.xz → ...Data/exampleDatasetAsTsv/gene_11.fasta.xz
diff --git a/testBaseData/exampleDataset/gene_2.fasta → ...BaseData/exampleDatasetAsTsv/gene_2.fasta b/testBaseData/exampleDataset/gene_2.fasta → ...BaseData/exampleDatasetAsTsv/gene_2.fasta
diff --git a/testBaseData/exampleDataset/gene_3.fasta → ...BaseData/exampleDatasetAsTsv/gene_3.fasta b/testBaseData/exampleDataset/gene_3.fasta → ...BaseData/exampleDatasetAsTsv/gene_3.fasta
diff --git a/testBaseData/exampleDataset/gene_4.fasta → ...BaseData/exampleDatasetAsTsv/gene_4.fasta b/testBaseData/exampleDataset/gene_4.fasta → ...BaseData/exampleDatasetAsTsv/gene_4.fasta
diff --git a/testBaseData/exampleDataset/gene_5.fasta → ...BaseData/exampleDatasetAsTsv/gene_5.fasta b/testBaseData/exampleDataset/gene_5.fasta → ...BaseData/exampleDatasetAsTsv/gene_5.fasta
diff --git a/testBaseData/exampleDataset/gene_6.fasta → ...BaseData/exampleDatasetAsTsv/gene_6.fasta b/testBaseData/exampleDataset/gene_6.fasta → ...BaseData/exampleDatasetAsTsv/gene_6.fasta
diff --git a/testBaseData/exampleDataset/gene_7.fasta → ...BaseData/exampleDatasetAsTsv/gene_7.fasta b/testBaseData/exampleDataset/gene_7.fasta → ...BaseData/exampleDatasetAsTsv/gene_7.fasta
diff --git a/testBaseData/exampleDataset/gene_8.fasta → ...BaseData/exampleDatasetAsTsv/gene_8.fasta b/testBaseData/exampleDataset/gene_8.fasta → ...BaseData/exampleDatasetAsTsv/gene_8.fasta
diff --git a/testBaseData/exampleDataset/gene_9.fasta → ...BaseData/exampleDatasetAsTsv/gene_9.fasta b/testBaseData/exampleDataset/gene_9.fasta → ...BaseData/exampleDatasetAsTsv/gene_9.fasta
diff --git a/...eDatasetAsNdjson/lineage_definitions.yaml → ...mpleDatasetAsTsv/lineage_definitions.yaml b/...eDatasetAsNdjson/lineage_definitions.yaml → ...mpleDatasetAsTsv/lineage_definitions.yaml
diff --git a/testBaseData/exampleDataset/nuc_0.fasta.zst → ...eData/exampleDatasetAsTsv/nuc_0.fasta.zst b/testBaseData/exampleDataset/nuc_0.fasta.zst → ...eData/exampleDatasetAsTsv/nuc_0.fasta.zst
diff --git a/testBaseData/exampleDataset/nuc_1.fasta → testBaseData/exampleDatasetAsTsv/nuc_1.fasta b/testBaseData/exampleDataset/nuc_1.fasta → testBaseData/exampleDatasetAsTsv/nuc_1.fasta
diff --git a/...aseData/exampleDataset/nuc_insertions.tsv → ...ta/exampleDatasetAsTsv/nuc_insertions.tsv b/...aseData/exampleDataset/nuc_insertions.tsv → ...ta/exampleDatasetAsTsv/nuc_insertions.tsv
diff --git a/...DatasetAsNdjson/preprocessing_config.yaml → ...pleDatasetAsTsv/preprocessing_config.yaml b/...DatasetAsNdjson/preprocessing_config.yaml → ...pleDatasetAsTsv/preprocessing_config.yaml
@@ -1,4 +1,3 @@
-ndjsonInputFilename: "input_file.ndjson"
+metadataFilename: "small_metadata_set.tsv"
 lineageDefinitionsFilename: "lineage_definitions.yaml"
 referenceGenomeFilename: "reference_genomes.json"
-duckdbMemoryLimitInG: 20
diff --git a/...pleDatasetAsNdjson/reference_genomes.json → ...xampleDatasetAsTsv/reference_genomes.json b/...pleDatasetAsNdjson/reference_genomes.json → ...xampleDatasetAsTsv/reference_genomes.json
diff --git a/...ata/exampleDataset/small_metadata_set.tsv → ...xampleDatasetAsTsv/small_metadata_set.tsv b/...ata/exampleDataset/small_metadata_set.tsv → ...xampleDatasetAsTsv/small_metadata_set.tsv
diff --git a/...Data/exampleDataset/unaligned_0.fasta.zst → ...exampleDatasetAsTsv/unaligned_0.fasta.zst b/...Data/exampleDataset/unaligned_0.fasta.zst → ...exampleDatasetAsTsv/unaligned_0.fasta.zst
diff --git a/...BaseData/exampleDataset/unaligned_1.fasta → ...ata/exampleDatasetAsTsv/unaligned_1.fasta b/...BaseData/exampleDataset/unaligned_1.fasta → ...ata/exampleDatasetAsTsv/unaligned_1.fasta
diff --git a/testBaseData/test_preprocessing_config.yaml b/testBaseData/test_preprocessing_config.yaml
@@ -1,4 +1,4 @@
-inputDirectory: "./testBaseData/exampleDataset/"
+inputDirectory: "./testBaseData/exampleDatasetAsTsv/"
 outputDirectory: "./output/"
 metadataFilename: "small_metadata_set.tsv"
 lineageDefinitionsFilename: "lineage_definitions.yaml"

diff --git a/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml b/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml
@@ -1,4 +1,4 @@
-inputDirectory: "./testBaseData/exampleDataset/"
+inputDirectory: "./testBaseData/exampleDatasetAsTsv/"
 outputDirectory: "./output/custom/"
 intermediateResultsDirectory: "./output/overriddenTemp/"
 metadataFilename: "small_metadata_set.tsv"