diff --git a/prql-compiler/src/sql/dialect.rs b/prql-compiler/src/sql/dialect.rs index e3b7359e7ec0..64199e54739d 100644 --- a/prql-compiler/src/sql/dialect.rs +++ b/prql-compiler/src/sql/dialect.rs @@ -81,11 +81,12 @@ impl Dialect { | Dialect::SQLite | Dialect::Postgres | Dialect::MySql - | Dialect::MsSql => SupportLevel::Supported, + | Dialect::MsSql + | Dialect::ClickHouse => SupportLevel::Supported, Dialect::Generic | Dialect::Ansi | Dialect::BigQuery | Dialect::Snowflake => { SupportLevel::Unsupported } - Dialect::Hive | Dialect::ClickHouse => SupportLevel::Nascent, + Dialect::Hive => SupportLevel::Nascent, } } diff --git a/prql-compiler/src/sql/std.sql.prql b/prql-compiler/src/sql/std.sql.prql index 593ef347d4e3..548c0e7d9ef9 100644 --- a/prql-compiler/src/sql/std.sql.prql +++ b/prql-compiler/src/sql/std.sql.prql @@ -114,6 +114,8 @@ module clickhouse { # https://clickhouse.com/docs/en/sql-reference/functions/arithmetic-functions#divide @{binding_strength=11} let div_f = l r -> s"({l} / {r})" + + let regex_search = text pattern -> s"match({text:0}, {pattern:0})" } module duckdb { diff --git a/prql-compiler/tests/integration/docker-compose.yml b/prql-compiler/tests/integration/docker-compose.yml index f339c1f93adf..09b10a873d15 100644 --- a/prql-compiler/tests/integration/docker-compose.yml +++ b/prql-compiler/tests/integration/docker-compose.yml @@ -46,3 +46,18 @@ services: LC_ALL: en_US.UTF-8 MSSQL_COLLATION: Latin1_General_100_CS_AI_SC_UTF8 volumes: *vol + clickhouse: + image: "clickhouse/clickhouse-server" + ports: + # 9004 is MySQL emulation port + # https://clickhouse.com/docs/en/guides/sre/network-ports + - "9004:9004" + environment: + CLICKHOUSE_DB: dummy + # Skip `chown` to user_files_path + # https://github.com/ClickHouse/ClickHouse/blob/01c7d2fe719f9b9ed59fce58d5e9dec44167e42f/docker/server/entrypoint.sh#L7-L9 + CLICKHOUSE_DO_NOT_CHOWN: "1" + volumes: + # ClickHouse can load csv only from user_files_path (default `/var/lib/clickhouse/user_files/`) + # https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#server_configuration_parameters-user_scripts_path + - ./data/chinook:/var/lib/clickhouse/user_files/chinook/:ro diff --git a/prql-compiler/tests/integration/main.rs b/prql-compiler/tests/integration/main.rs index 810ccf7a12e7..a692d018287c 100644 --- a/prql-compiler/tests/integration/main.rs +++ b/prql-compiler/tests/integration/main.rs @@ -106,6 +106,13 @@ impl IntegrationTest for Dialect { ), }), #[cfg(feature = "test-external-dbs")] + Dialect::ClickHouse => Some(DbConnection { + dialect: Dialect::ClickHouse, + protocol: Box::new( + mysql::Pool::new("mysql://default:@localhost:9004/dummy").unwrap(), + ), + }), + #[cfg(feature = "test-external-dbs")] Dialect::MsSql => { use tiberius::{AuthMethod, Client, Config}; use tokio::net::TcpStream; @@ -210,6 +217,15 @@ impl IntegrationTest for Dialect { fs::remove_file(&new_path).unwrap(); query_result.unwrap(); } + Dialect::ClickHouse => { + protocol.run_query( + &format!( + "INSERT INTO {csv_name} SELECT * FROM file('/var/lib/clickhouse/user_files/chinook/{csv_name}.csv')" + ), + runtime, + ) + .unwrap(); + } Dialect::MsSql => { protocol.run_query(&format!("BULK INSERT {csv_name} FROM '/tmp/chinook/{csv_name}.csv' WITH (FIRSTROW = 2, FIELDTERMINATOR = ',', ROWTERMINATOR = '\n', TABLOCK, FORMAT = 'CSV', CODEPAGE = 'RAW');"), runtime).unwrap(); } @@ -221,6 +237,13 @@ impl IntegrationTest for Dialect { Dialect::DuckDb => sql.replace("REAL", "DOUBLE"), Dialect::Postgres => sql.replace("REAL", "DOUBLE PRECISION"), Dialect::MySql => sql.replace("TIMESTAMP", "DATETIME"), + Dialect::ClickHouse => { + let re = Regex::new(r"(?s)\)$").unwrap(); + re.replace(&sql, r") ENGINE = Memory") + .replace("TIMESTAMP", "DATETIME64") + .replace("REAL", "DOUBLE") + .replace("VARCHAR(255)", "Nullable(String)") + } Dialect::MsSql => sql .replace("TIMESTAMP", "DATETIME") .replace("REAL", "FLOAT(53)") diff --git a/prql-compiler/tests/integration/queries/arithmetic.prql b/prql-compiler/tests/integration/queries/arithmetic.prql index 90f9a6c41203..bc23fff6fe9e 100644 --- a/prql-compiler/tests/integration/queries/arithmetic.prql +++ b/prql-compiler/tests/integration/queries/arithmetic.prql @@ -1,3 +1,4 @@ +# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785) from [ { x_int = 13, x_float = 13.0, k_int = 5, k_float = 5.0 }, { x_int = -13, x_float = -13.0, k_int = 5, k_float = 5.0 }, diff --git a/prql-compiler/tests/integration/queries/genre_counts.prql b/prql-compiler/tests/integration/queries/genre_counts.prql index 58ad3d8e080c..4271867f853e 100644 --- a/prql-compiler/tests/integration/queries/genre_counts.prql +++ b/prql-compiler/tests/integration/queries/genre_counts.prql @@ -1,3 +1,4 @@ +# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785) let genre_count = ( from genres aggregate {a = count name} diff --git a/prql-compiler/tests/integration/queries/invoice_totals.prql b/prql-compiler/tests/integration/queries/invoice_totals.prql index a9f77ecd2172..123627c4c4ce 100644 --- a/prql-compiler/tests/integration/queries/invoice_totals.prql +++ b/prql-compiler/tests/integration/queries/invoice_totals.prql @@ -1,3 +1,4 @@ +# skip_clickhouse (clickhouse doesn't have lag function) # skip_mssql (error: The function 'LAG' may not have a window frame.) from i=invoices join ii=invoice_items (==invoice_id) diff --git a/prql-compiler/tests/integration/queries/loop.prql b/prql-compiler/tests/integration/queries/loop.prql index 80f660e9bf33..82fa9c8f327b 100644 --- a/prql-compiler/tests/integration/queries/loop.prql +++ b/prql-compiler/tests/integration/queries/loop.prql @@ -1,3 +1,4 @@ +# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785) # skip_mssql (the keyword RECURSIVE is not allowed and you have to declare the columns for CTE) from [{n = 1}] select n = n - 2 diff --git a/prql-compiler/tests/integration/queries/set_ops_remove.prql b/prql-compiler/tests/integration/queries/set_ops_remove.prql index 787d171e53ca..5302d9528fcf 100644 --- a/prql-compiler/tests/integration/queries/set_ops_remove.prql +++ b/prql-compiler/tests/integration/queries/set_ops_remove.prql @@ -1,3 +1,4 @@ +# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785) let distinct = rel -> (from t = _param.rel | group {t.*} (take 1)) from_text format:json '{ "columns": ["a"], "data": [[1], [2], [2], [3]] }'