From d097b41b6bb15006e9644ab5e714432377452345 Mon Sep 17 00:00:00 2001 From: sotirangelo Date: Fri, 26 Jan 2024 15:27:13 +0200 Subject: [PATCH] refactor(load): move datasources from dvmql to load dir --- src/dvmql/datasources.rs | 97 ++++++++++++++++------------------------ src/load/mod.rs | 51 +++++++++++++++++++++ 2 files changed, 90 insertions(+), 58 deletions(-) diff --git a/src/dvmql/datasources.rs b/src/dvmql/datasources.rs index 501627d..d6dbf5d 100644 --- a/src/dvmql/datasources.rs +++ b/src/dvmql/datasources.rs @@ -7,6 +7,8 @@ use log::{debug, info, trace}; use serde::Deserialize; use std::collections::HashMap; +use crate::load::{Csv, Database, Datasource, Excel, Xml}; + use super::helpers::read_xml_file; /// Helper function for loading and deserializing the datasources XML file @@ -20,7 +22,7 @@ pub fn load_datasources_xml(datasources_path: String) -> Result Datasource::Csv(Csv::from(init_ds)), - "xml" => Datasource::Xml(Csv::from(init_ds)), + "xml" => Datasource::Xml(Xml::from(init_ds)), "db" => Datasource::Database(Database::from(init_ds)), "excel" => Datasource::Excel(Excel::from(init_ds)), // TODO: Handle error @@ -54,7 +56,7 @@ struct DeserializedDatasource { filename: Option, path: Option, sheet: Option, - delimiter: Option, + delimiter: Option, headings: Option, system: Option, connection: Option, @@ -63,25 +65,7 @@ struct DeserializedDatasource { database: Option, } -/// Enum representing the different types of datasources. -#[derive(Debug, PartialEq)] -pub enum Datasource { - Csv(Csv), - Xml(Csv), - Excel(Excel), - Database(Database), -} - -/// CSV datasource (Also used for XML for now) -#[derive(Debug, PartialEq)] -pub struct Csv { - id: u8, - name: String, - filename: String, - path: String, - delimiter: String, - headings: String, -} +const HEADINGS_HAYSTACK: [&str; 4] = ["yes", "true", "y", "1"]; impl From<&DeserializedDatasource> for Csv { fn from(ds: &DeserializedDatasource) -> Csv { @@ -100,24 +84,33 @@ impl From<&DeserializedDatasource> for Csv { .delimiter .to_owned() .expect("CSV delimiter field should be defined"), - headings: ds - .headings - .to_owned() - .expect("CSV headings field should be defined"), + has_headers: HEADINGS_HAYSTACK.contains( + &ds.headings + .to_owned() + .expect("CSV headings should be defined") + .as_str(), + ), } } } -/// Excel datasource -#[derive(Debug, PartialEq)] -pub struct Excel { - id: u8, - name: String, - filename: String, - path: String, - sheet: String, - headings: String, +impl From<&DeserializedDatasource> for Xml { + fn from(ds: &DeserializedDatasource) -> Xml { + Xml { + id: ds.id, + name: ds.name.to_owned(), + filename: ds + .filename + .to_owned() + .expect("CSV filename field should be defined"), + path: ds + .path + .to_owned() + .expect("CSV path field should be defined"), + } + } } + impl From<&DeserializedDatasource> for Excel { fn from(ds: &DeserializedDatasource) -> Excel { Excel { @@ -135,26 +128,16 @@ impl From<&DeserializedDatasource> for Excel { .sheet .to_owned() .expect("Excel sheet field should be defined"), - headings: ds - .headings - .to_owned() - .expect("Excel headings field should be defined"), + has_headers: HEADINGS_HAYSTACK.contains( + &ds.headings + .to_owned() + .expect("Excel headings should be defined") + .as_str(), + ), } } } -/// Database datasource -#[derive(Debug, PartialEq)] -pub struct Database { - id: u8, - name: String, - system: String, - connection: String, - username: String, - password: String, - database: String, -} - impl From<&DeserializedDatasource> for Database { fn from(ds: &DeserializedDatasource) -> Database { Database { @@ -199,7 +182,7 @@ mod tests { name: String::from("myCSV"), filename: Some(String::from("file.csv")), path: Some(String::from("/some-path/")), - delimiter: Some(String::from(",")), + delimiter: Some(','), headings: Some(String::from("yes")), sheet: None, system: None, @@ -229,7 +212,7 @@ mod tests { name: String::from("myCSV2"), filename: Some(String::from("file.xml")), path: Some(String::from("/some-path/")), - delimiter: Some(String::from(",")), + delimiter: Some(','), headings: Some(String::from("yes")), sheet: None, system: None, @@ -264,8 +247,8 @@ mod tests { name: String::from("myCSV"), filename: String::from("file.csv"), path: String::from("/some-path/"), - delimiter: String::from(","), - headings: String::from("yes"), + delimiter: ',', + has_headers: true, }), Datasource::Excel(Excel { id: 2, @@ -273,15 +256,13 @@ mod tests { filename: String::from("file.xlsx"), path: String::from("/some-path/"), sheet: String::from("Sheet1"), - headings: String::from("no"), + has_headers: false, }), - Datasource::Xml(Csv { + Datasource::Xml(Xml { id: 3, name: String::from("myCSV2"), filename: String::from("file.xml"), path: String::from("/some-path/"), - delimiter: String::from(","), - headings: String::from("yes"), }), Datasource::Database(Database { id: 4, diff --git a/src/load/mod.rs b/src/load/mod.rs index baed2ef..9d9087a 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -1,2 +1,53 @@ pub mod edges; +/// Enum representing the different types of datasources. +#[derive(Debug, PartialEq)] +pub enum Datasource { + Csv(Csv), + Xml(Xml), + Excel(Excel), + Database(Database), +} + +/// CSV datasource +#[derive(Debug, PartialEq)] +pub struct Csv { + pub id: u8, + pub name: String, + pub filename: String, + pub path: String, + pub delimiter: char, + pub has_headers: bool, +} + +/// XML datasource +#[derive(Debug, PartialEq)] +pub struct Xml { + pub id: u8, + pub name: String, + pub filename: String, + pub path: String, +} + +/// Database datasource +#[derive(Debug, PartialEq)] +pub struct Database { + pub id: u8, + pub name: String, + pub system: String, + pub connection: String, + pub username: String, + pub password: String, + pub database: String, +} + +/// Excel datasource +#[derive(Debug, PartialEq)] +pub struct Excel { + pub id: u8, + pub name: String, + pub filename: String, + pub path: String, + pub sheet: String, + pub has_headers: bool, +}