test: clickhouse integration tests

prql-compiler/src/sql/dialect.rs

@@ -81,11 +81,12 @@ impl Dialect {
             | Dialect::SQLite
             | Dialect::Postgres
             | Dialect::MySql
-            | Dialect::MsSql => SupportLevel::Supported,
+            | Dialect::MsSql
+            | Dialect::ClickHouse => SupportLevel::Supported,
             Dialect::Generic | Dialect::Ansi | Dialect::BigQuery | Dialect::Snowflake => {
                 SupportLevel::Unsupported
             }
-            Dialect::Hive | Dialect::ClickHouse => SupportLevel::Nascent,
+            Dialect::Hive => SupportLevel::Nascent,
         }
     }
 

prql-compiler/src/sql/std.sql.prql

@@ -114,6 +114,8 @@ module clickhouse {
   # https://clickhouse.com/docs/en/sql-reference/functions/arithmetic-functions#divide
   @{binding_strength=11}
   let div_f = l r -> s"({l} / {r})"
+
+  let regex_search = text pattern -> s"match({text:0}, {pattern:0})"
 }
 
 module duckdb {

prql-compiler/tests/integration/docker-compose.yml

@@ -46,3 +46,18 @@ services:
       LC_ALL: en_US.UTF-8
       MSSQL_COLLATION: Latin1_General_100_CS_AI_SC_UTF8
     volumes: *vol
+  clickhouse:
+    image: "clickhouse/clickhouse-server"
+    ports:
+      # 9004 is MySQL emulation port
+      # https://clickhouse.com/docs/en/guides/sre/network-ports
+      - "9004:9004"
+    environment:
+      CLICKHOUSE_DB: dummy
+      # Skip `chown` to user_files_path
+      # https://github.com/ClickHouse/ClickHouse/blob/01c7d2fe719f9b9ed59fce58d5e9dec44167e42f/docker/server/entrypoint.sh#L7-L9
+      CLICKHOUSE_DO_NOT_CHOWN: "1"
+    volumes:
+      # ClickHouse can load csv only from user_files_path (default `/var/lib/clickhouse/user_files/`)
+      # https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#server_configuration_parameters-user_scripts_path
+      - ./data/chinook:/var/lib/clickhouse/user_files/chinook/:ro

prql-compiler/tests/integration/main.rs

@@ -106,6 +106,13 @@ impl IntegrationTest for Dialect {
                 ),
             }),
             #[cfg(feature = "test-external-dbs")]
+            Dialect::ClickHouse => Some(DbConnection {
+                dialect: Dialect::ClickHouse,
+                protocol: Box::new(
+                    mysql::Pool::new("mysql://default:@localhost:9004/dummy").unwrap(),
+                ),
+            }),
+            #[cfg(feature = "test-external-dbs")]
             Dialect::MsSql => {
                 use tiberius::{AuthMethod, Client, Config};
                 use tokio::net::TcpStream;
@@ -210,6 +217,15 @@ impl IntegrationTest for Dialect {
                 fs::remove_file(&new_path).unwrap();
                 query_result.unwrap();
             }
+            Dialect::ClickHouse => {
+                protocol.run_query(
+                    &format!(
+                        "INSERT INTO {csv_name} SELECT * FROM file('/var/lib/clickhouse/user_files/chinook/{csv_name}.csv')"
+                    ),
+                    runtime,
+                )
+                .unwrap();
+            }
             Dialect::MsSql => {
                 protocol.run_query(&format!("BULK INSERT {csv_name} FROM '/tmp/chinook/{csv_name}.csv' WITH (FIRSTROW = 2, FIELDTERMINATOR = ',', ROWTERMINATOR = '\n', TABLOCK, FORMAT = 'CSV', CODEPAGE = 'RAW');"), runtime).unwrap();
             }
@@ -221,6 +237,13 @@ impl IntegrationTest for Dialect {
             Dialect::DuckDb => sql.replace("REAL", "DOUBLE"),
             Dialect::Postgres => sql.replace("REAL", "DOUBLE PRECISION"),
             Dialect::MySql => sql.replace("TIMESTAMP", "DATETIME"),
+            Dialect::ClickHouse => {
+                let re = Regex::new(r"(?s)\)$").unwrap();
+                re.replace(&sql, r") ENGINE = Memory")
+                    .replace("TIMESTAMP", "DATETIME64")
+                    .replace("REAL", "DOUBLE")
+                    .replace("VARCHAR(255)", "Nullable(String)")
+            }
             Dialect::MsSql => sql
                 .replace("TIMESTAMP", "DATETIME")
                 .replace("REAL", "FLOAT(53)")

prql-compiler/tests/integration/queries/arithmetic.prql

@@ -1,3 +1,4 @@
+# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785)
 from [
     { x_int =  13, x_float =  13.0, k_int =  5, k_float =  5.0 },
     { x_int = -13, x_float = -13.0, k_int =  5, k_float =  5.0 },

prql-compiler/tests/integration/queries/genre_counts.prql

@@ -1,3 +1,4 @@
+# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785)
 let genre_count = (
     from genres
     aggregate {a = count name}

prql-compiler/tests/integration/queries/invoice_totals.prql

@@ -1,3 +1,4 @@
+# skip_clickhouse (clickhouse doesn't have lag function)
 # skip_mssql (error: The function 'LAG' may not have a window frame.)
 from i=invoices
 join ii=invoice_items (==invoice_id)

prql-compiler/tests/integration/queries/loop.prql

@@ -1,3 +1,4 @@
+# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785)
 # skip_mssql (the keyword RECURSIVE is not allowed and you have to declare the columns for CTE)
 from [{n = 1}]
 select n = n - 2

prql-compiler/tests/integration/queries/set_ops_remove.prql

@@ -1,3 +1,4 @@
+# skip_clickhouse (https://github.com/PRQL/prql/pull/2815#issuecomment-1587496785)
 let distinct = rel -> (from t = _param.rel | group {t.*} (take 1))
 
 from_text format:json '{ "columns": ["a"], "data": [[1], [2], [2], [3]] }'