Redesigned preprocessing

include/silo/preprocessing/preprocessing_database.h

@@ -14,6 +14,11 @@ namespace preprocessing {
 class Partitions;
 
 class PreprocessingDatabase {
+  public:
+   static constexpr std::string_view COMPRESS_NUC = "compressNuc";
+   static constexpr std::string_view COMPRESS_AA = "compressAA";
+
+  private:
    duckdb::DuckDB duck_db;
    duckdb::Connection connection;
 
@@ -24,7 +29,7 @@ class PreprocessingDatabase {
 
    Partitions getPartitionDescriptor();
 
-   void registerSequences(const silo::ReferenceGenomes& reference_genomes);
+   static void registerSequences(const silo::ReferenceGenomes& reference_genomes);
 
    std::unique_ptr<duckdb::MaterializedQueryResult> query(std::string sql_query);
 
@@ -35,5 +40,11 @@ class PreprocessingDatabase {
    );
 };
 
+std::vector<std::string> extractStringListValue(
+   duckdb::MaterializedQueryResult& result,
+   size_t row,
+   size_t column
+);
+
 }  // namespace preprocessing
 }  // namespace silo

src/silo/preprocessing/metadata_info.cpp

@@ -6,29 +6,13 @@
 #include <duckdb.hpp>
 
 #include "silo/config/database_config.h"
+#include "silo/preprocessing/preprocessing_database.h"
 #include "silo/preprocessing/preprocessing_exception.h"
 
 namespace {
 
-std::vector<std::string> extractStringListValue(
-   duckdb::MaterializedQueryResult& result,
-   size_t row,
-   size_t column
-) {
-   std::vector<std::string> return_value;
-   const duckdb::Value tmp_value = result.GetValue(column, row);
-   std::vector<duckdb::Value> child_values = duckdb::ListValue::GetChildren(tmp_value);
-   std::transform(
-      child_values.begin(),
-      child_values.end(),
-      std::back_inserter(return_value),
-      [](const duckdb::Value& value) { return value.GetValue<std::string>(); }
-   );
-   return return_value;
-}
-
 std::unordered_map<std::string, std::string> validateFieldsAgainstConfig(
-   const std::unordered_map<std::string, std::string> found_metadata_fields,
+   const std::unordered_map<std::string, std::string>& found_metadata_fields,
    const silo::config::DatabaseConfig& database_config
 ) {
    std::vector<std::string> config_metadata_fields;
@@ -40,10 +24,10 @@ std::unordered_map<std::string, std::string> validateFieldsAgainstConfig(
    );
 
    std::unordered_map<std::string, std::string> validated_metadata_fields;
-   for (const auto& pair : found_metadata_fields) {
-      if (std::find(config_metadata_fields.begin(), config_metadata_fields.end(), pair.first)
+   for (const auto& [field_name, access_path] : found_metadata_fields) {
+      if (std::find(config_metadata_fields.begin(), config_metadata_fields.end(), field_name)
        != config_metadata_fields.end()) {
-         validated_metadata_fields.emplace(pair);
+         validated_metadata_fields.emplace(field_name, access_path);
       }
    }
    for (const std::string& name : config_metadata_fields) {
@@ -57,8 +41,8 @@ std::unordered_map<std::string, std::string> validateFieldsAgainstConfig(
    }
 
    std::string metadata_field_string;
-   for (const auto& [field, select] : validated_metadata_fields) {
-      metadata_field_string += "'" + field + "' with selection '" + select + "',";
+   for (const auto& [field_name, select] : validated_metadata_fields) {
+      metadata_field_string += "'" + field_name + "' with selection '" + select + "',";
    }
    SPDLOG_TRACE("Found metadata fields: " + metadata_field_string);
    return validated_metadata_fields;
@@ -167,7 +151,7 @@ MetadataInfo MetadataInfo::validateFromNdjsonFile(
    }
 
    std::unordered_map<std::string, std::string> metadata_fields_to_validate;
-   for (const std::string& metadata_field : extractStringListValue(*result, 0, 0)) {
+   for (const std::string& metadata_field : preprocessing::extractStringListValue(*result, 0, 0)) {
       metadata_fields_to_validate[metadata_field] = "metadata." + metadata_field;
    }
    detectInsertionLists(ndjson_file, metadata_fields_to_validate);

src/silo/preprocessing/preprocessing_database.cpp

@@ -119,14 +119,14 @@ PreprocessingDatabase::PreprocessingDatabase(const std::string& backing_file)
    }
 
    connection.CreateVectorizedFunction(
-      "compressNuc",
+      std::string(COMPRESS_NUC),
       {duckdb::LogicalType::VARCHAR, duckdb::LogicalType::VARCHAR},
       duckdb::LogicalType::BLOB,
       Compressors::compressNuc
    );
 
    connection.CreateVectorizedFunction(
-      "compressAA",
+      std::string(COMPRESS_AA),
       {duckdb::LogicalType::VARCHAR, duckdb::LogicalType::VARCHAR},
       duckdb::LogicalType::BLOB,
       Compressors::compressAA
@@ -161,7 +161,7 @@ preprocessing::Partitions PreprocessingDatabase::getPartitionDescriptor() {
         it != partition_descriptor_from_sql->end();
         ++it) {
       const duckdb::Value db_partition_id = it.current_row.GetValue<duckdb::Value>(0);
-      int64_t partition_id_int = duckdb::BigIntValue::Get(db_partition_id);
+      const int64_t partition_id_int = duckdb::BigIntValue::Get(db_partition_id);
       if (partition_id_int != check_partition_id_sorted_and_contiguous) {
          throw PreprocessingException(
             "The partition IDs produced by the preprocessing are not sorted, not starting from 0 "
@@ -172,7 +172,7 @@ preprocessing::Partitions PreprocessingDatabase::getPartitionDescriptor() {
       uint32_t partition_id = partition_id_int;
 
       const duckdb::Value db_partition_size = it.current_row.GetValue<duckdb::Value>(1);
-      int64_t partition_size_bigint = duckdb::BigIntValue::Get(db_partition_size);
+      const int64_t partition_size_bigint = duckdb::BigIntValue::Get(db_partition_size);
       if (partition_size_bigint <= 0) {
          throw PreprocessingException("Non-positive partition size encountered.");
       }
@@ -182,7 +182,7 @@ preprocessing::Partitions PreprocessingDatabase::getPartitionDescriptor() {
          );
       }
 
-      uint32_t partition_size = static_cast<uint32_t>(partition_size_bigint);
+      const auto partition_size = static_cast<uint32_t>(partition_size_bigint);
 
       partitions.emplace_back(std::vector<preprocessing::PartitionChunk>{
          {partition_id, 0, partition_size, 0}});
@@ -200,4 +200,21 @@ void PreprocessingDatabase::generateNucSequenceTable(
    ZstdFastaTable::generate(connection, table_name, fasta_reader, reference_sequence);
 }
 
+std::vector<std::string> extractStringListValue(
+   duckdb::MaterializedQueryResult& result,
+   size_t row,
+   size_t column
+) {
+   std::vector<std::string> return_value;
+   const duckdb::Value tmp_value = result.GetValue(column, row);
+   std::vector<duckdb::Value> child_values = duckdb::ListValue::GetChildren(tmp_value);
+   std::transform(
+      child_values.begin(),
+      child_values.end(),
+      std::back_inserter(return_value),
+      [](const duckdb::Value& value) { return value.GetValue<std::string>(); }
+   );
+   return return_value;
+}
+
 }  // namespace silo::preprocessing

src/silo/preprocessing/sequence_info.cpp

@@ -7,25 +7,6 @@
 #include "silo/preprocessing/preprocessing_exception.h"
 #include "silo/storage/reference_genomes.h"
 
-namespace {
-std::vector<std::string> extractStringListValue(
-   duckdb::MaterializedQueryResult& result,
-   size_t row,
-   size_t column
-) {
-   std::vector<std::string> return_value;
-   const duckdb::Value tmp_value = result.GetValue(column, row);
-   std::vector<duckdb::Value> child_values = duckdb::ListValue::GetChildren(tmp_value);
-   std::transform(
-      child_values.begin(),
-      child_values.end(),
-      std::back_inserter(return_value),
-      [](const duckdb::Value& value) { return value.GetValue<std::string>(); }
-   );
-   return return_value;
-}
-}  // namespace
-
 namespace silo::preprocessing {
 
 SequenceInfo::SequenceInfo(const silo::ReferenceGenomes& reference_genomes) {
@@ -49,15 +30,17 @@ std::vector<std::string> SequenceInfo::getSequenceSelects() {
    std::vector<std::string> sequence_selects;
    for (const std::string& name : nuc_sequence_names) {
       sequence_selects.emplace_back(fmt::format(
-         "compressNuc(alignedNucleotideSequences.{0}, "
-         "'{0}') as nuc_{0}",
+         "{0}(alignedNucleotideSequences.{1}, "
+         "'{1}') as nuc_{1}",
+         preprocessing::PreprocessingDatabase::COMPRESS_NUC,
          name
       ));
    }
    for (const std::string& name : aa_sequence_names) {
       sequence_selects.emplace_back(fmt::format(
-         "compressAA(alignedAminoAcidSequences.{0}, "
-         "'{0}') as gene_{0}",
+         "{0}(alignedAminoAcidSequences.{1}, "
+         "'{1}') as gene_{1}",
+         preprocessing::PreprocessingDatabase::COMPRESS_AA,
          name
       ));
    }