From 62c8c797e281a4dc808a0ae31d90a71ab984a719 Mon Sep 17 00:00:00 2001 From: Simon Sawert Date: Tue, 7 Jan 2025 10:45:20 +0100 Subject: [PATCH 1/5] chore: Rebase --- src/ast/dml.rs | 28 +++++++-- src/ast/query.rs | 15 ++++- src/ast/spans.rs | 2 + src/dialect/clickhouse.rs | 8 +++ src/keywords.rs | 2 +- src/parser/mod.rs | 103 +++++++++++++++++++++++++--------- tests/sqlparser_clickhouse.rs | 26 ++++++++- tests/sqlparser_postgres.rs | 10 +++- 8 files changed, 156 insertions(+), 38 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index f64818e61..489c73887 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -33,10 +33,10 @@ pub use super::ddl::{ColumnDef, TableConstraint}; use super::{ display_comma_separated, display_separated, Assignment, ClusteredBy, CommentDef, Expr, - FileFormat, FromTable, HiveDistributionStyle, HiveFormat, HiveIOFormat, HiveRowFormat, Ident, - InsertAliases, MysqlInsertPriority, ObjectName, OnCommit, OnInsert, OneOrManyWithParens, - OrderByExpr, Query, RowAccessPolicy, SelectItem, SqlOption, SqliteOnConflict, TableEngine, - TableWithJoins, Tag, WrappedCollection, + FileFormat, FormatClause, FromTable, HiveDistributionStyle, HiveFormat, HiveIOFormat, + HiveRowFormat, Ident, InsertAliases, MysqlInsertPriority, ObjectName, OnCommit, OnInsert, + OneOrManyWithParens, OrderByExpr, Query, RowAccessPolicy, SelectItem, Setting, SqlOption, + SqliteOnConflict, TableEngine, TableWithJoins, Tag, WrappedCollection, }; /// CREATE INDEX statement. @@ -498,6 +498,20 @@ pub struct Insert { pub priority: Option, /// Only for mysql pub insert_alias: Option, + /// Settings used in together with a specified `FORMAT`. + /// + /// ClickHouse syntax: `INSERT INTO tbl SETTINGS format_template_resultset = '/some/path/resultset.format'` + /// + /// [ClickHouse `INSERT INTO`](https://clickhouse.com/docs/en/sql-reference/statements/insert-into) + /// [ClickHouse Formats](https://clickhouse.com/docs/en/interfaces/formats) + pub settings: Option>, + /// Format for `INSERT` statement when not using standard SQL format. Can be e.g. `CSV`, + /// `JSON`, `JSONAsString`, `LineAsString` and more. + /// + /// ClickHouse syntax: `INSERT INTO tbl FORMAT JSONEachRow {"foo": 1, "bar": 2}, {"foo": 3}` + /// + /// [ClickHouse formats JSON insert](https://clickhouse.com/docs/en/interfaces/formats#json-inserting-data) + pub format_clause: Option, } impl Display for Insert { @@ -546,11 +560,17 @@ impl Display for Insert { write!(f, "({}) ", display_comma_separated(&self.after_columns))?; } + if let Some(settings) = &self.settings { + write!(f, "SETTINGS {} ", display_comma_separated(settings))?; + } + if let Some(source) = &self.source { write!(f, "{source}")?; } else if !self.assignments.is_empty() { write!(f, "SET ")?; write!(f, "{}", display_comma_separated(&self.assignments))?; + } else if let Some(format_clause) = &self.format_clause { + write!(f, "{format_clause}")?; } else if self.source.is_none() && self.columns.is_empty() { write!(f, "DEFAULT VALUES")?; } diff --git a/src/ast/query.rs b/src/ast/query.rs index 9e4e9e2ef..f4101c3d2 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -2465,14 +2465,25 @@ impl fmt::Display for GroupByExpr { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum FormatClause { - Identifier(Ident), + Identifier { + ident: Ident, + expr: Option>, + }, Null, } impl fmt::Display for FormatClause { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - FormatClause::Identifier(ident) => write!(f, "FORMAT {}", ident), + FormatClause::Identifier { ident, expr } => { + write!(f, "FORMAT {}", ident)?; + + if let Some(exprs) = expr { + write!(f, " {}", display_comma_separated(exprs))?; + } + + Ok(()) + } FormatClause::Null => write!(f, "FORMAT NULL"), } } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 2ca659147..cb741e214 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -1154,6 +1154,8 @@ impl Spanned for Insert { priority: _, // todo, mysql specific insert_alias: _, // todo, mysql specific assignments, + settings: _, // todo, clickhouse specific + format_clause: _, // todo, clickhouse specific } = self; union_spans( diff --git a/src/dialect/clickhouse.rs b/src/dialect/clickhouse.rs index 0c8f08040..87ba85a15 100644 --- a/src/dialect/clickhouse.rs +++ b/src/dialect/clickhouse.rs @@ -50,4 +50,12 @@ impl Dialect for ClickHouseDialect { fn supports_limit_comma(&self) -> bool { true } + + // ClickHouse uses this for some FORMAT expressions in `INSERT` context, e.g. when inserting + // with FORMAT JSONEachRow a raw JSON key-value expression is valid and expected. + // + // [ClickHouse formats](https://clickhouse.com/docs/en/interfaces/formats) + fn supports_dictionary_syntax(&self) -> bool { + true + } } diff --git a/src/keywords.rs b/src/keywords.rs index b7ff39e04..367fb1a21 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -936,7 +936,7 @@ pub const RESERVED_FOR_TABLE_ALIAS: &[Keyword] = &[ Keyword::PREWHERE, // for ClickHouse SELECT * FROM t SETTINGS ... Keyword::SETTINGS, - // for ClickHouse SELECT * FROM t FORMAT... + // for ClickHouse SELECT * FROM t FORMAT... or INSERT INTO t FORMAT... Keyword::FORMAT, // for Snowflake START WITH .. CONNECT BY Keyword::START, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 85ae66399..48f593747 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -9589,12 +9589,7 @@ impl<'a> Parser<'a> { let format_clause = if dialect_of!(self is ClickHouseDialect | GenericDialect) && self.parse_keyword(Keyword::FORMAT) { - if self.parse_keyword(Keyword::NULL) { - Some(FormatClause::Null) - } else { - let ident = self.parse_identifier()?; - Some(FormatClause::Identifier(ident)) - } + Some(self.parse_format_clause(false)?) } else { None }; @@ -11899,35 +11894,56 @@ impl<'a> Parser<'a> { let is_mysql = dialect_of!(self is MySqlDialect); - let (columns, partitioned, after_columns, source, assignments) = - if self.parse_keywords(&[Keyword::DEFAULT, Keyword::VALUES]) { - (vec![], None, vec![], None, vec![]) - } else { - let (columns, partitioned, after_columns) = if !self.peek_subquery_start() { - let columns = self.parse_parenthesized_column_list(Optional, is_mysql)?; + let (columns, partitioned, after_columns, source, assignments) = if self + .parse_keywords(&[Keyword::DEFAULT, Keyword::VALUES]) + { + (vec![], None, vec![], None, vec![]) + } else { + let (columns, partitioned, after_columns) = if !self.peek_subquery_start() { + let columns = self.parse_parenthesized_column_list(Optional, is_mysql)?; - let partitioned = self.parse_insert_partition()?; - // Hive allows you to specify columns after partitions as well if you want. - let after_columns = if dialect_of!(self is HiveDialect) { - self.parse_parenthesized_column_list(Optional, false)? - } else { - vec![] - }; - (columns, partitioned, after_columns) + let partitioned = self.parse_insert_partition()?; + // Hive allows you to specify columns after partitions as well if you want. + let after_columns = if dialect_of!(self is HiveDialect) { + self.parse_parenthesized_column_list(Optional, false)? } else { - Default::default() + vec![] }; + (columns, partitioned, after_columns) + } else { + Default::default() + }; - let (source, assignments) = - if self.dialect.supports_insert_set() && self.parse_keyword(Keyword::SET) { - (None, self.parse_comma_separated(Parser::parse_assignment)?) - } else { - (Some(self.parse_query()?), vec![]) - }; + let (source, assignments) = if self.peek_keyword(Keyword::FORMAT) + || self.peek_keyword(Keyword::SETTINGS) + { + (None, vec![]) + } else if self.dialect.supports_insert_set() && self.parse_keyword(Keyword::SET) { + (None, self.parse_comma_separated(Parser::parse_assignment)?) + } else { + (Some(self.parse_query()?), vec![]) + }; + + (columns, partitioned, after_columns, source, assignments) + }; + + let (format_clause, settings) = if dialect_of!(self is ClickHouseDialect | GenericDialect) + { + // Settings always comes before `FORMAT` for ClickHouse: + // + let settings = self.parse_settings()?; - (columns, partitioned, after_columns, source, assignments) + let format = if self.parse_keyword(Keyword::FORMAT) { + Some(self.parse_format_clause(true)?) + } else { + None }; + (format, settings) + } else { + (None, None) + }; + let insert_alias = if dialect_of!(self is MySqlDialect | GenericDialect) && self.parse_keyword(Keyword::AS) { @@ -12012,10 +12028,41 @@ impl<'a> Parser<'a> { replace_into, priority, insert_alias, + settings, + format_clause, })) } } + // Parses format clause used for [ClickHouse]. Formats are different when using `SELECT` and + // `INSERT` and also when using the CLI for pipes. It may or may not take an additional + // expression after the format so we try to parse the expression but allow failure. + // + // Since we know we never take an additional expression in `SELECT` context we never only try + // to parse if `can_have_expression` is true. + // + // + pub fn parse_format_clause( + &mut self, + can_have_expression: bool, + ) -> Result { + if self.parse_keyword(Keyword::NULL) { + Ok(FormatClause::Null) + } else { + let ident = self.parse_identifier()?; + let expr = if can_have_expression { + match self.try_parse(|p| p.parse_comma_separated(|p| p.parse_expr())) { + Ok(expr) => Some(expr), + _ => None, + } + } else { + None + }; + + Ok(FormatClause::Identifier { ident, expr }) + } + } + /// Returns true if the immediate tokens look like the /// beginning of a subquery. `(SELECT ...` fn peek_subquery_start(&mut self) -> bool { diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index 2f1b043b6..ec4470737 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -1378,7 +1378,10 @@ fn test_query_with_format_clause() { } else { assert_eq!( query.format_clause, - Some(FormatClause::Identifier(Ident::new(*format))) + Some(FormatClause::Identifier { + ident: Ident::new(*format), + expr: None + }) ); } } @@ -1398,6 +1401,27 @@ fn test_query_with_format_clause() { } } +#[test] +fn test_insert_query_with_format_clause() { + let cases = [ + r#"INSERT INTO tbl FORMAT JSONEachRow {"id": 1, "value": "foo"}, {"id": 2, "value": "bar"}"#, + r#"INSERT INTO tbl FORMAT JSONEachRow ["first", "second", "third"]"#, + r#"INSERT INTO tbl FORMAT JSONEachRow [{"first": 1}]"#, + r#"INSERT INTO tbl FORMAT jsoneachrow {"id": 1}"#, + r#"INSERT INTO tbl (foo) FORMAT JSONAsObject {"foo": {"bar": {"x": "y"}, "baz": 1}}"#, + r#"INSERT INTO tbl (foo, bar) FORMAT JSON {"foo": 1, "bar": 2}"#, + r#"INSERT INTO tbl FORMAT CSV col1, col2, col3"#, + r#"INSERT INTO tbl FORMAT LineAsString "I love apple", "I love banana", "I love orange""#, + r#"INSERT INTO tbl (foo) SETTINGS input_format_json_read_bools_as_numbers = true FORMAT JSONEachRow {"id": 1, "value": "foo"}"#, + r#"INSERT INTO tbl SETTINGS format_template_resultset = '/some/path/resultset.format', format_template_row = '/some/path/row.format' FORMAT Template"#, + r#"INSERT INTO tbl SETTINGS input_format_json_read_bools_as_numbers = true FORMAT JSONEachRow {"id": 1, "value": "foo"}"#, + ]; + + for sql in &cases { + clickhouse_and_generic().verified_stmt(sql); + } +} + #[test] fn parse_create_table_on_commit_and_as_query() { let sql = r#"CREATE LOCAL TEMPORARY TABLE test ON COMMIT PRESERVE ROWS AS SELECT 1"#; diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 1a621ee74..2a2516ffa 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -4431,7 +4431,9 @@ fn test_simple_postgres_insert_with_alias() { returning: None, replace_into: false, priority: None, - insert_alias: None + insert_alias: None, + settings: None, + format_clause: None, }) ) } @@ -4502,7 +4504,9 @@ fn test_simple_postgres_insert_with_alias() { returning: None, replace_into: false, priority: None, - insert_alias: None + insert_alias: None, + settings: None, + format_clause: None, }) ) } @@ -4570,6 +4574,8 @@ fn test_simple_insert_with_quoted_alias() { replace_into: false, priority: None, insert_alias: None, + settings: None, + format_clause: None, }) ) } From 79308bfa2f9db0fdb6fc01ae1fc1f1023f8cdd49 Mon Sep 17 00:00:00 2001 From: Simon Sawert Date: Tue, 7 Jan 2025 10:46:59 +0100 Subject: [PATCH 2/5] doc: Remove keyword documentation --- src/keywords.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/keywords.rs b/src/keywords.rs index 367fb1a21..1e6f1b56e 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -934,9 +934,7 @@ pub const RESERVED_FOR_TABLE_ALIAS: &[Keyword] = &[ Keyword::PARTITION, // for Clickhouse PREWHERE Keyword::PREWHERE, - // for ClickHouse SELECT * FROM t SETTINGS ... Keyword::SETTINGS, - // for ClickHouse SELECT * FROM t FORMAT... or INSERT INTO t FORMAT... Keyword::FORMAT, // for Snowflake START WITH .. CONNECT BY Keyword::START, From 182d44c0934b3436419031b02f5e6d1b43b66ad5 Mon Sep 17 00:00:00 2001 From: Simon Sawert Date: Tue, 7 Jan 2025 14:26:19 +0100 Subject: [PATCH 3/5] docs: Simplify docs --- src/ast/dml.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 489c73887..319c20c6b 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -498,12 +498,11 @@ pub struct Insert { pub priority: Option, /// Only for mysql pub insert_alias: Option, - /// Settings used in together with a specified `FORMAT`. + /// Settings used for ClickHouse. /// /// ClickHouse syntax: `INSERT INTO tbl SETTINGS format_template_resultset = '/some/path/resultset.format'` /// /// [ClickHouse `INSERT INTO`](https://clickhouse.com/docs/en/sql-reference/statements/insert-into) - /// [ClickHouse Formats](https://clickhouse.com/docs/en/interfaces/formats) pub settings: Option>, /// Format for `INSERT` statement when not using standard SQL format. Can be e.g. `CSV`, /// `JSON`, `JSONAsString`, `LineAsString` and more. From 062c2b4e4d4cea5b11e907eac5d3c0c6f4759259 Mon Sep 17 00:00:00 2001 From: Simon Sawert Date: Tue, 7 Jan 2025 14:26:43 +0100 Subject: [PATCH 4/5] test: Dedupe test cases for case insensitivity --- tests/sqlparser_clickhouse.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index ec4470737..5771d5e58 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -1407,7 +1407,6 @@ fn test_insert_query_with_format_clause() { r#"INSERT INTO tbl FORMAT JSONEachRow {"id": 1, "value": "foo"}, {"id": 2, "value": "bar"}"#, r#"INSERT INTO tbl FORMAT JSONEachRow ["first", "second", "third"]"#, r#"INSERT INTO tbl FORMAT JSONEachRow [{"first": 1}]"#, - r#"INSERT INTO tbl FORMAT jsoneachrow {"id": 1}"#, r#"INSERT INTO tbl (foo) FORMAT JSONAsObject {"foo": {"bar": {"x": "y"}, "baz": 1}}"#, r#"INSERT INTO tbl (foo, bar) FORMAT JSON {"foo": 1, "bar": 2}"#, r#"INSERT INTO tbl FORMAT CSV col1, col2, col3"#, From e75aafec3017dcd71cb8959fdd65dcc7ed94f33f Mon Sep 17 00:00:00 2001 From: Simon Sawert Date: Tue, 7 Jan 2025 14:32:36 +0100 Subject: [PATCH 5/5] fix: Remove `source` check in `columns` branch If `source` is not `None` we will land in the first if branch so no need to check if it's `None` again. --- src/ast/dml.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/dml.rs b/src/ast/dml.rs index 319c20c6b..3a76e7db2 100644 --- a/src/ast/dml.rs +++ b/src/ast/dml.rs @@ -570,7 +570,7 @@ impl Display for Insert { write!(f, "{}", display_comma_separated(&self.assignments))?; } else if let Some(format_clause) = &self.format_clause { write!(f, "{format_clause}")?; - } else if self.source.is_none() && self.columns.is_empty() { + } else if self.columns.is_empty() { write!(f, "DEFAULT VALUES")?; }