fix: correctly escape quotes in field names

include/silo/preprocessing/identifiers.h

@@ -11,8 +11,15 @@ class Identifiers {
    std::vector<Identifier> identifiers;
 
   public:
+   explicit Identifiers() = default;
+
    Identifiers(const std::vector<std::string>& raw_identifiers);
 
+   template <typename T>
+   void addIdentifier(T&& identifier) {
+      identifiers.emplace_back(std::forward<T>(identifier));
+   }
+
    Identifiers prefix(const std::string& prefix) const;
 
    size_t size() const;

include/silo/preprocessing/metadata_info.h

@@ -5,6 +5,7 @@
 #include <vector>
 
 #include "silo/config/database_config.h"
+#include "silo/preprocessing/identifiers.h"
 
 namespace silo::preprocessing {
 
@@ -21,7 +22,7 @@ class MetadataInfo {
       const silo::config::DatabaseConfig& database_config
    );
 
-   static std::vector<std::string> getMetadataFields(
+   static silo::preprocessing::Identifiers getMetadataFields(
       const silo::config::DatabaseConfig& database_config
    );
 

src/silo/preprocessing/metadata_info.cpp

@@ -112,8 +112,8 @@ void MetadataInfo::validateNdjsonFile(
 
    auto result = connection.Query(fmt::format(
       "SELECT metadata.* "
-      "FROM read_json_auto(\"{}\") LIMIT 0; ",
-      ndjson_file.string()
+      "FROM read_json_auto({}) LIMIT 0; ",
+      Identifier{ndjson_file.string()}.escape()
    ));
 
    if (result->HasError()) {
@@ -157,13 +157,10 @@ void MetadataInfo::validateNdjsonFile(
    }
 }
 
-std::vector<std::string> MetadataInfo::getMetadataFields(
-   const silo::config::DatabaseConfig& database_config
-) {
-   std::vector<std::string> ret;
-   ret.reserve(database_config.schema.metadata.size());
+Identifiers MetadataInfo::getMetadataFields(const silo::config::DatabaseConfig& database_config) {
+   Identifiers ret;
    for (const auto& field : database_config.schema.metadata) {
-      ret.push_back("\"" + field.name + "\"");
+      ret.addIdentifier(field.name);
    }
    return ret;
 }
@@ -174,7 +171,7 @@ std::vector<std::string> MetadataInfo::getMetadataSQLTypes(
    std::vector<std::string> ret;
    ret.reserve(database_config.schema.metadata.size());
    for (const auto& field : database_config.schema.metadata) {
-      ret.push_back(fmt::format("\"{}\" {}", field.name, toSQLType(field.type)));
+      ret.push_back(fmt::format("{} {}", Identifier{field.name}.escape(), toSQLType(field.type)));
    }
    return ret;
 }
@@ -185,7 +182,7 @@ std::vector<std::string> MetadataInfo::getMetadataSelects(
    std::vector<std::string> ret;
    ret.reserve(database_config.schema.metadata.size());
    for (const auto& field : database_config.schema.metadata) {
-      ret.push_back(fmt::format(R"( "metadata"."{0}" AS "{0}")", field.name));
+      ret.push_back(fmt::format(R"( "metadata".{0} AS {0})", Identifier{field.name}.escape()));
    }
    return ret;
 }

src/silo/preprocessing/metadata_info.test.cpp

@@ -53,7 +53,8 @@ TEST(MetadataInfo, isValidMedataFileShouldReturnTrueWithValidMetadataFile) {
          }
    };
 
-   const auto fields = silo::preprocessing::MetadataInfo::getMetadataFields(valid_config);
+   const auto fields =
+      silo::preprocessing::MetadataInfo::getMetadataFields(valid_config).getRawIdentifierStrings();
    ASSERT_TRUE(std::ranges::find(fields, R"("gisaid_epi_isl")") != fields.end());
    ASSERT_TRUE(std::ranges::find(fields, R"("pango_lineage")") != fields.end());
    ASSERT_TRUE(std::ranges::find(fields, R"("date")") != fields.end());
@@ -77,8 +78,8 @@ TEST(MetadataInfo, shouldValidateCorrectNdjsonInputFile) {
          }
    };
 
-   const auto fields = silo::preprocessing::MetadataInfo::getMetadataFields(valid_config);
-
+   const auto fields =
+      silo::preprocessing::MetadataInfo::getMetadataFields(valid_config).getRawIdentifierStrings();
    ASSERT_TRUE(std::ranges::find(fields, R"("gisaid_epi_isl")") != fields.end());
    ASSERT_TRUE(std::ranges::find(fields, R"("pango_lineage")") != fields.end());
    ASSERT_TRUE(std::ranges::find(fields, R"("date")") != fields.end());

src/silo/preprocessing/preprocessor.cpp

@@ -230,7 +230,9 @@ void Preprocessor::buildMetadataTableFromFile(const std::filesystem::path& metad
    (void)preprocessing_db.query(fmt::format(
       "INSERT INTO metadata_table BY NAME (SELECT {} FROM read_csv_auto('{}', delim = '\t', "
       "header = true));",
-      boost::join(MetadataInfo::getMetadataFields(database_config), ","),
+      boost::join(
+         MetadataInfo::getMetadataFields(database_config).getEscapedIdentifierStrings(), ","
+      ),
       metadata_filename.string()
    ));
 }

src/silo/preprocessing/preprocessor.test.cpp

@@ -142,6 +142,30 @@ const Scenario TSV_FILE_WITH_SQL_KEYWORD_AS_FIELD = {
    .expected_query_result = NDJSON_WITH_SQL_KEYWORD_AS_FIELD.expected_query_result
 };
 
+const Scenario TSV_FILE_WITH_QUOTE_IN_FIELD_NAME = {
+   .input_directory = "testBaseData/tsvWithQuoteInFieldName/",
+   .expected_sequence_count = 2,
+   .query = R"(
+{
+   "action": {
+      "type": "Aggregated",
+      "groupByFields": ["x\"y"],
+      "orderByFields": ["x\"y"]
+   },
+   "filterExpression": {
+      "type": "StringEquals",
+      "column": "x\"y",
+      "value": "a"
+   }
+}
+   )",
+   .expected_query_result = nlohmann::json::parse(
+      R"([
+         {"count": 1, "x\"y": "a"}
+   ])"
+   )
+};
+
 const Scenario EMPTY_INPUT_TSV = {
    .input_directory = "testBaseData/emptyInputTsv/",
    .expected_sequence_count = 0,
@@ -288,6 +312,7 @@ INSTANTIATE_TEST_SUITE_P(
       SAM_FILES,
       NDJSON_WITH_SQL_KEYWORD_AS_FIELD,
       TSV_FILE_WITH_SQL_KEYWORD_AS_FIELD,
+      TSV_FILE_WITH_QUOTE_IN_FIELD_NAME,
       NDJSON_WITH_NUMERIC_NAMES,
       EMPTY_INPUT_TSV,
       EMPTY_INPUT_NDJSON,

testBaseData/tsvWithQuoteInFieldName/aa_insertions.tsv

@@ -0,0 +1 @@
+id

testBaseData/tsvWithQuoteInFieldName/database_config.yaml

@@ -0,0 +1,8 @@
+schema:
+  instanceName: Test
+  metadata:
+    - name: id
+      type: string
+    - name: 'x"y'
+      type: string
+  primaryKey: id

testBaseData/tsvWithQuoteInFieldName/metadata.tsv

@@ -0,0 +1,3 @@
+id	"x""y"
+1	a
+2	b

testBaseData/tsvWithQuoteInFieldName/nuc_insertions.tsv

@@ -0,0 +1 @@
+id

testBaseData/tsvWithQuoteInFieldName/preprocessing_config.yaml

@@ -0,0 +1,2 @@
+metadataFilename: "metadata.tsv"
+referenceGenomeFilename: "reference_genomes.json"

testBaseData/tsvWithQuoteInFieldName/reference_genomes.json

@@ -0,0 +1,4 @@
+{
+  "nucleotideSequences": [],
+  "genes": []
+}