Last edits

GenSpectrum · Dec 18, 2023 · 619035b · 619035b
1 parent fabf265
commit 619035b
Show file tree

Hide file tree

Showing 8 changed files with 2 additions and 39 deletions.
diff --git a/include/silo/preprocessing/preprocessing_config.h b/include/silo/preprocessing/preprocessing_config.h
@@ -62,16 +62,6 @@ struct GenePrefix {
 };
 const GenePrefix DEFAULT_GENE_PREFIX = {"gene_"};
 
-struct PartitionsFolder {
-   std::string folder;
-};
-const PartitionsFolder DEFAULT_PARTITIONS_FOLDER = {"partitions/"};
-
-struct SortedPartitionsFolder {
-   std::string folder;
-};
-const SortedPartitionsFolder DEFAULT_SORTED_PARTITIONS_FOLDER = {"partitions_sorted/"};
-
 struct ReferenceGenomeFilename {
    std::string filename;
 };
@@ -102,8 +92,6 @@ class PreprocessingConfig {
       const NdjsonInputFilename& ndjson_input_filename_,
       const MetadataFilename& metadata_filename_,
       const PangoLineageDefinitionFilename& pango_lineage_definition_filename_,
-      const PartitionsFolder& partition_folder_,
-      const SortedPartitionsFolder& sorted_partition_folder_,
       const ReferenceGenomeFilename& reference_genome_filename_,
       const NucleotideSequencePrefix& nucleotide_sequence_prefix_,
       const GenePrefix& gene_prefix_

diff --git a/include/silo/preprocessing/preprocessing_config_reader.h b/include/silo/preprocessing/preprocessing_config_reader.h
@@ -39,14 +39,6 @@ struct OptionalPreprocessingConfig {
     * The filename of the pango lineage definition file, relative to the inputDirectory
     */
    std::optional<std::filesystem::path> pango_lineage_definition_file;
-   /**
-    * Folder for intermediate partition files, relative to the intermediateResultsDirectory
-    */
-   std::optional<std::filesystem::path> partition_folder;
-   /**
-    * Folder for intermediate sorted partition files, relative to the intermediateResultsDirectory
-    */
-   std::optional<std::filesystem::path> sorted_partition_folder;
    /**
     * The filename of the reference genome file, relative to the inputDirectory
     */

diff --git a/include/silo/zstdfasta/zstdfasta_table_reader.h b/include/silo/zstdfasta/zstdfasta_table_reader.h
@@ -27,8 +27,6 @@ class ZstdFastaTableReader {
    std::unique_ptr<silo::ZstdDecompressor> decompressor;
    size_t current_row;
 
-   // std::string DEBUG_dictionary;
-
    std::string genome_buffer;
 
    std::optional<std::string> nextKey();

diff --git a/src/silo/preprocessing/metadata_info.cpp b/src/silo/preprocessing/metadata_info.cpp
@@ -126,9 +126,8 @@ MetadataInfo MetadataInfo::validateFromNdjsonFile(
    duckdb::Connection connection(db);
 
    auto result = connection.Query(fmt::format(
-      "SELECT json_keys(metadata)"
-      "FROM "
-      "'{}' LIMIT 1; ",
+      "SELECT json_keys(metadata) "
+      "FROM read_json_auto(\"{}\") LIMIT 1; ",
       ndjson_file.string()
    ));
    if (result->HasError()) {

diff --git a/src/silo/preprocessing/preprocessing_config.cpp b/src/silo/preprocessing/preprocessing_config.cpp
@@ -30,8 +30,6 @@ PreprocessingConfig::PreprocessingConfig(
    const NdjsonInputFilename& ndjson_input_filename_,
    const MetadataFilename& metadata_filename_,
    const PangoLineageDefinitionFilename& pango_lineage_definition_filename_,
-   const PartitionsFolder& partition_folder_,
-   const SortedPartitionsFolder& sorted_partition_folder_,
    const ReferenceGenomeFilename& reference_genome_filename_,
    const NucleotideSequencePrefix& nucleotide_sequence_prefix_,
    const GenePrefix& gene_prefix_

diff --git a/src/silo/preprocessing/preprocessing_config_reader.cpp b/src/silo/preprocessing/preprocessing_config_reader.cpp
@@ -31,8 +31,6 @@ struct convert<OptionalPreprocessingConfig> {
          extractStringIfPresent(node, "ndjsonInputFilename"),
          extractStringIfPresent(node, "metadataFilename"),
          extractStringIfPresent(node, "pangoLineageDefinitionFilename"),
-         extractStringIfPresent(node, "partitionsFolder"),
-         extractStringIfPresent(node, "sortedPartitionsFolder"),
          extractStringIfPresent(node, "referenceGenomeFilename"),
          extractStringIfPresent(node, "nucleotideSequencePrefix"),
          extractStringIfPresent(node, "genePrefix")};
@@ -92,13 +90,6 @@ PreprocessingConfig OptionalPreprocessingConfig::mergeValuesFromOrDefault(
       PangoLineageDefinitionFilename{
          pango_lineage_definition_file.has_value() ? pango_lineage_definition_file
                                                    : other.pango_lineage_definition_file},
-      PartitionsFolder{partition_folder.value_or(
-         other.partition_folder.value_or(silo::preprocessing::DEFAULT_PARTITIONS_FOLDER.folder)
-      )},
-      SortedPartitionsFolder{
-         sorted_partition_folder.value_or(other.sorted_partition_folder.value_or(
-            silo::preprocessing::DEFAULT_SORTED_PARTITIONS_FOLDER.folder
-         ))},
       ReferenceGenomeFilename{reference_genome_file.value_or(other.reference_genome_file.value_or(
          silo::preprocessing::DEFAULT_REFERENCE_GENOME_FILENAME.filename
       ))},

diff --git a/src/silo/preprocessing/preprocessing_config_reader.test.cpp b/src/silo/preprocessing/preprocessing_config_reader.test.cpp
@@ -44,7 +44,6 @@ TEST(PreprocessingConfigReader, shouldReadConfigWithOverriddenDefaults) {
    );
 
    const std::string input_directory = "./testBaseData/exampleDataset/";
-   const std::string intermediate_directory = "./output/overriddenTemp/";
    ASSERT_EQ(config.getMetadataInputFilename(), input_directory + "small_metadata_set.tsv");
    ASSERT_EQ(
       config.getPangoLineageDefinitionFilename(), input_directory + "pangolineage_alias.json"

diff --git a/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml b/testBaseData/test_preprocessing_config_with_overridden_defaults.yaml
@@ -4,7 +4,5 @@ intermediateResultsDirectory: "./output/overriddenTemp/"
 metadataFilename: "small_metadata_set.tsv"
 pangoLineageDefinitionFilename: "pangolineage_alias.json"
 referenceGenomeFilename: "reference_genomes.json"
-partitionsFolder: "folder1/"
-sortedPartitionsFolder: "folder2/"
 genePrefix: "aaSeq_"
 nucleotideSequencePrefix: ""