From 822cb9d2d7c5c621669cc32bcae6d74fea31b412 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 15:51:24 +0800 Subject: [PATCH 01/10] fix(ast): fix incorrect display for ast --- src/meta/app/src/principal/file_format.rs | 6 - .../app/src/principal/principal_identity.rs | 2 +- src/query/ast/src/ast/common.rs | 16 +- src/query/ast/src/ast/format/ast_format.rs | 27 +- src/query/ast/src/ast/format/syntax/dml.rs | 9 +- src/query/ast/src/ast/query.rs | 648 +++++++------- src/query/ast/src/ast/statements/catalog.rs | 6 +- .../ast/src/ast/statements/connection.rs | 33 +- src/query/ast/src/ast/statements/copy.rs | 88 +- src/query/ast/src/ast/statements/database.rs | 8 +- src/query/ast/src/ast/statements/explain.rs | 6 +- src/query/ast/src/ast/statements/index.rs | 12 +- src/query/ast/src/ast/statements/insert.rs | 10 +- .../ast/src/ast/statements/merge_into.rs | 9 +- .../ast/src/ast/statements/notification.rs | 11 +- src/query/ast/src/ast/statements/presign.rs | 4 +- src/query/ast/src/ast/statements/share.rs | 5 +- src/query/ast/src/ast/statements/stage.rs | 94 +-- src/query/ast/src/ast/statements/statement.rs | 48 +- src/query/ast/src/ast/statements/table.rs | 41 +- src/query/ast/src/ast/statements/task.rs | 121 +-- src/query/ast/src/ast/statements/udf.rs | 42 +- src/query/ast/src/ast/statements/update.rs | 12 +- src/query/ast/src/ast/statements/user.rs | 12 +- src/query/ast/src/ast/visitors/visitor.rs | 3 +- src/query/ast/src/ast/visitors/visitor_mut.rs | 3 +- src/query/ast/src/parser/stage.rs | 54 +- src/query/ast/src/parser/statement.rs | 68 +- src/query/ast/tests/it/parser.rs | 1 + src/query/ast/tests/it/testdata/statement.txt | 792 ++++++++++++------ src/query/sql/src/planner/binder/binder.rs | 2 +- src/query/sql/src/planner/binder/ddl/stage.rs | 12 +- src/query/sql/src/planner/binder/ddl/task.rs | 2 +- src/query/sql/src/planner/binder/explain.rs | 18 +- src/query/sql/src/planner/binder/insert.rs | 3 +- src/query/sql/src/planner/binder/replace.rs | 3 +- .../sql/src/planner/plans/ddl/notification.rs | 2 +- src/query/sql/src/planner/plans/ddl/task.rs | 2 +- 38 files changed, 1283 insertions(+), 952 deletions(-) diff --git a/src/meta/app/src/principal/file_format.rs b/src/meta/app/src/principal/file_format.rs index 55d9e1ab7d65c..6305da900f6ee 100644 --- a/src/meta/app/src/principal/file_format.rs +++ b/src/meta/app/src/principal/file_format.rs @@ -50,12 +50,6 @@ pub struct FileFormatOptionsAst { pub options: BTreeMap, } -impl Display for FileFormatOptionsAst { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.options) - } -} - impl FileFormatOptionsAst { pub fn new(options: BTreeMap) -> Self { FileFormatOptionsAst { options } diff --git a/src/meta/app/src/principal/principal_identity.rs b/src/meta/app/src/principal/principal_identity.rs index f3d425c0c972c..d89445c353242 100644 --- a/src/meta/app/src/principal/principal_identity.rs +++ b/src/meta/app/src/principal/principal_identity.rs @@ -36,7 +36,7 @@ impl fmt::Display for PrincipalIdentity { fn fmt(&self, f: &mut fmt::Formatter) -> std::result::Result<(), fmt::Error> { match self { PrincipalIdentity::User(u) => write!(f, " USER {u}"), - PrincipalIdentity::Role(r) => write!(f, " ROLE {r}"), + PrincipalIdentity::Role(r) => write!(f, " ROLE '{r}'"), } } } diff --git a/src/query/ast/src/ast/common.rs b/src/query/ast/src/ast/common.rs index b0eac3e5a1259..80400566af703 100644 --- a/src/query/ast/src/ast/common.rs +++ b/src/query/ast/src/ast/common.rs @@ -216,7 +216,7 @@ pub(crate) fn write_comma_separated_list( } /// Write input items into `'a', 'b', 'c'` -pub(crate) fn write_comma_separated_quoted_list( +pub(crate) fn write_comma_separated_string_list( f: &mut Formatter<'_>, items: impl IntoIterator, ) -> std::fmt::Result { @@ -233,6 +233,20 @@ pub(crate) fn write_comma_separated_quoted_list( pub(crate) fn write_comma_separated_map( f: &mut Formatter<'_>, items: impl IntoIterator, +) -> std::fmt::Result { + for (i, (k, v)) in items.into_iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{k} = {v}")?; + } + Ok(()) +} + +/// Write input map items into `field_a='x', field_b='y'` +pub(crate) fn write_comma_separated_string_map( + f: &mut Formatter<'_>, + items: impl IntoIterator, ) -> std::fmt::Result { for (i, (k, v)) in items.into_iter().enumerate() { if i > 0 { diff --git a/src/query/ast/src/ast/format/ast_format.rs b/src/query/ast/src/ast/format/ast_format.rs index a6e674dddb236..dc579436b8110 100644 --- a/src/query/ast/src/ast/format/ast_format.rs +++ b/src/query/ast/src/ast/format/ast_format.rs @@ -733,12 +733,11 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { "({})", options .iter() - .flat_map(|opt| { + .map(|opt| { match opt { - ExplainOption::Verbose(true) => Some("Verbose"), - ExplainOption::Logical(true) => Some("Logical"), - ExplainOption::Optimized(true) => Some("Optimized"), - _ => None, + ExplainOption::Verbose => "Verbose", + ExplainOption::Logical => "Logical", + ExplainOption::Optimized => "Optimized", } }) .join(", ") @@ -816,8 +815,8 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { children.push(pattern_node); } if !copy.file_format.is_empty() { - let mut file_formats_children = Vec::with_capacity(copy.file_format.len()); - for (k, v) in copy.file_format.iter() { + let mut file_formats_children = Vec::new(); + for (k, v) in copy.file_format.options.iter() { let file_format_name = format!("FileFormat {} = {:?}", k, v); let file_format_format_ctx = AstFormatContext::new(file_format_name); let file_format_node = FormatTreeNode::new(file_format_format_ctx); @@ -899,9 +898,9 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { ); children.push(from_node); - if !copy.file_format.is_empty() { - let mut file_formats_children = Vec::with_capacity(copy.file_format.len()); - for (k, v) in copy.file_format.iter() { + if !copy.file_format.options.is_empty() { + let mut file_formats_children = Vec::with_capacity(copy.file_format.options.len()); + for (k, v) in copy.file_format.options.iter() { let file_format_name = format!("FileFormat {} = {:?}", k, v); let file_format_format_ctx = AstFormatContext::new(file_format_name); let file_format_node = FormatTreeNode::new(file_format_format_ctx); @@ -1095,8 +1094,8 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { self.children.push(streaming_node); } InsertSource::StreamingV2 { settings, .. } => { - let mut file_formats_children = Vec::with_capacity(settings.len()); - for (k, v) in settings.iter() { + let mut file_formats_children = Vec::new(); + for (k, v) in settings.options.iter() { let file_format_name = format!("FileFormat {} = {:?}", k, v); let file_format_format_ctx = AstFormatContext::new(file_format_name); let file_format_node = FormatTreeNode::new(file_format_format_ctx); @@ -2480,8 +2479,8 @@ impl<'ast> Visitor<'ast> for AstFormatVisitor { children.push(FormatTreeNode::new(location_format_ctx)); } if !stmt.file_format_options.is_empty() { - let mut file_formats_children = Vec::with_capacity(stmt.file_format_options.len()); - for (k, v) in stmt.file_format_options.iter() { + let mut file_formats_children = Vec::new(); + for (k, v) in stmt.file_format_options.options.iter() { let file_format_name = format!("FileFormat {} = {:?}", k, v); let file_format_format_ctx = AstFormatContext::new(file_format_name); let file_format_node = FormatTreeNode::new(file_format_format_ctx); diff --git a/src/query/ast/src/ast/format/syntax/dml.rs b/src/query/ast/src/ast/format/syntax/dml.rs index 8601a8a7fb69b..552781bd58473 100644 --- a/src/query/ast/src/ast/format/syntax/dml.rs +++ b/src/query/ast/src/ast/format/syntax/dml.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; - use pretty::RcDoc; use super::expr::pretty_expr; @@ -28,6 +26,7 @@ use crate::ast::CopyIntoLocationStmt; use crate::ast::CopyIntoTableSource; use crate::ast::CopyIntoTableStmt; use crate::ast::DeleteStmt; +use crate::ast::FileFormatOptions; use crate::ast::InsertSource; use crate::ast::InsertStmt; use crate::ast::UpdateExpr; @@ -95,7 +94,7 @@ fn pretty_source(source: InsertSource) -> RcDoc<'static> { RcDoc::line() .append(RcDoc::text("FILE_FORMAT_SETTINGS = ")) .append(parenthesized( - interweave_comma(settings.iter().map(|(k, v)| { + interweave_comma(settings.options.iter().map(|(k, v)| { RcDoc::text(k.to_string()) .append(RcDoc::space()) .append(RcDoc::text("=")) @@ -282,12 +281,12 @@ pub(crate) fn pretty_copy_into_location(copy_stmt: CopyIntoLocationStmt) -> RcDo ) } -fn pretty_file_format(file_format: &BTreeMap) -> RcDoc<'static> { +fn pretty_file_format(file_format: &FileFormatOptions) -> RcDoc<'static> { if !file_format.is_empty() { RcDoc::line() .append(RcDoc::text("FILE_FORMAT = ")) .append(parenthesized( - interweave_comma(file_format.iter().map(|(k, v)| { + interweave_comma(file_format.options.iter().map(|(k, v)| { RcDoc::text(k.to_string()) .append(RcDoc::space()) .append(RcDoc::text("=")) diff --git a/src/query/ast/src/ast/query.rs b/src/query/ast/src/ast/query.rs index 9821c5ab5a944..b3c94a93853ca 100644 --- a/src/query/ast/src/ast/query.rs +++ b/src/query/ast/src/ast/query.rs @@ -52,6 +52,37 @@ pub struct Query { pub ignore_result: bool, } +impl Display for Query { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // CTE, with clause + if let Some(with) = &self.with { + write!(f, "WITH {with} ")?; + } + + // Query body + write!(f, "{}", self.body)?; + + // ORDER BY clause + if !self.order_by.is_empty() { + write!(f, " ORDER BY ")?; + write_comma_separated_list(f, &self.order_by)?; + } + + // LIMIT clause + if !self.limit.is_empty() { + write!(f, " LIMIT ")?; + write_comma_separated_list(f, &self.limit)?; + } + + // TODO: We should validate if offset exists, limit should be empty or just one element + if let Some(offset) = &self.offset { + write!(f, " OFFSET {offset}")?; + } + + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct With { #[drive(skip)] @@ -61,6 +92,17 @@ pub struct With { pub ctes: Vec, } +impl Display for With { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.recursive { + write!(f, "RECURSIVE ")?; + } + + write_comma_separated_list(f, &self.ctes)?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct CTE { #[drive(skip)] @@ -71,6 +113,17 @@ pub struct CTE { pub query: Box, } +impl Display for CTE { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{} AS ", self.alias)?; + if self.materialized { + write!(f, "MATERIALIZED ")?; + } + write!(f, "({})", self.query)?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct SetOperation { #[drive(skip)] @@ -108,6 +161,73 @@ pub struct SelectStmt { pub qualify: Option, } +impl Display for SelectStmt { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // SELECT clause + write!(f, "SELECT ")?; + if let Some(hints) = &self.hints { + write!(f, "{} ", hints)?; + } + if self.distinct { + write!(f, "DISTINCT ")?; + } + write_comma_separated_list(f, &self.select_list)?; + + // FROM clause + if !self.from.is_empty() { + write!(f, " FROM ")?; + write_comma_separated_list(f, &self.from)?; + } + + // WHERE clause + if let Some(expr) = &self.selection { + write!(f, " WHERE {expr}")?; + } + + // GROUP BY clause + if self.group_by.is_some() { + write!(f, " GROUP BY ")?; + match self.group_by.as_ref().unwrap() { + GroupBy::Normal(exprs) => { + write_comma_separated_list(f, exprs)?; + } + GroupBy::All => { + write!(f, "ALL")?; + } + GroupBy::GroupingSets(sets) => { + write!(f, "GROUPING SETS (")?; + for (i, set) in sets.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "(")?; + write_comma_separated_list(f, set)?; + write!(f, ")")?; + } + write!(f, ")")?; + } + GroupBy::Cube(exprs) => { + write!(f, "CUBE (")?; + write_comma_separated_list(f, exprs)?; + write!(f, ")")?; + } + GroupBy::Rollup(exprs) => { + write!(f, "ROLLUP (")?; + write_comma_separated_list(f, exprs)?; + write!(f, ")")?; + } + } + } + + // HAVING clause + if let Some(having) = &self.having { + write!(f, " HAVING {having}")?; + } + + Ok(()) + } +} + /// Group by Clause. #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum GroupBy { @@ -140,6 +260,49 @@ pub enum SetExpr { }, } +impl Display for SetExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + SetExpr::Select(select_stmt) => { + write!(f, "{select_stmt}")?; + } + SetExpr::Query(query) => { + write!(f, "({query})")?; + } + SetExpr::SetOperation(set_operation) => { + write!(f, "{}", set_operation.left)?; + match set_operation.op { + SetOperator::Union => { + write!(f, " UNION ")?; + } + SetOperator::Except => { + write!(f, " EXCEPT ")?; + } + SetOperator::Intersect => { + write!(f, " INTERSECT ")?; + } + } + if set_operation.all { + write!(f, "ALL ")?; + } + write!(f, "{}", set_operation.right)?; + } + SetExpr::Values { values, .. } => { + write!(f, "VALUES")?; + for (i, value) in values.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "(")?; + write_comma_separated_list(f, value)?; + write!(f, ")")?; + } + } + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub enum SetOperator { Union, @@ -159,6 +322,27 @@ pub struct OrderByExpr { pub nulls_first: Option, } +impl Display for OrderByExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.expr)?; + if let Some(asc) = self.asc { + if asc { + write!(f, " ASC")?; + } else { + write!(f, " DESC")?; + } + } + if let Some(nulls_first) = self.nulls_first { + if nulls_first { + write!(f, " NULLS FIRST")?; + } else { + write!(f, " NULLS LAST")?; + } + } + Ok(()) + } +} + /// One item of the comma-separated list following `SELECT` #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum SelectTarget { @@ -220,6 +404,37 @@ impl SelectTarget { } } +impl Display for SelectTarget { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + SelectTarget::AliasedExpr { expr, alias } => { + write!(f, "{expr}")?; + if let Some(ident) = alias { + write!(f, " AS {ident}")?; + } + } + SelectTarget::StarColumns { + qualified, + column_filter, + } => match column_filter { + Some(ColumnFilter::Excludes(excludes)) => { + write_dot_separated_list(f, qualified)?; + write!(f, " EXCLUDE (")?; + write_comma_separated_list(f, excludes)?; + write!(f, ")")?; + } + Some(ColumnFilter::Lambda(lambda)) => { + write!(f, "COLUMNS({lambda})")?; + } + None => { + write_dot_separated_list(f, qualified)?; + } + }, + } + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum ColumnFilter { Excludes(Vec), @@ -256,6 +471,20 @@ pub enum Indirection { Star(#[drive(skip)] Span), } +impl Display for Indirection { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Indirection::Identifier(ident) => { + write!(f, "{ident}")?; + } + Indirection::Star(_) => { + write!(f, "*")?; + } + } + Ok(()) + } +} + /// Time Travel specification #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum TimeTravelPoint { @@ -263,6 +492,21 @@ pub enum TimeTravelPoint { Timestamp(Box), } +impl Display for TimeTravelPoint { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + TimeTravelPoint::Snapshot(sid) => { + write!(f, "(SNAPSHOT => '{sid}')")?; + } + TimeTravelPoint::Timestamp(ts) => { + write!(f, "(TIMESTAMP => {ts})")?; + } + } + + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct Pivot { pub aggregate: Expr, @@ -270,6 +514,15 @@ pub struct Pivot { pub values: Vec, } +impl Display for Pivot { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "PIVOT({} FOR {} IN (", self.aggregate, self.value_column)?; + write_comma_separated_list(f, &self.values)?; + write!(f, "))")?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub struct Unpivot { pub value_column: Identifier, @@ -277,11 +530,24 @@ pub struct Unpivot { pub names: Vec, } -/// A table name or a parenthesized subquery with an optional alias -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum TableReference { - // Table name - Table { +impl Display for Unpivot { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "UNPIVOT({} FOR {} IN (", + self.value_column, self.column_name + )?; + write_comma_separated_list(f, &self.names)?; + write!(f, "))")?; + Ok(()) + } +} + +/// A table name or a parenthesized subquery with an optional alias +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum TableReference { + // Table name + Table { #[drive(skip)] span: Span, catalog: Option, @@ -352,124 +618,6 @@ impl TableReference { } } -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub struct TableAlias { - pub name: Identifier, - pub columns: Vec, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct Join { - pub op: JoinOperator, - pub condition: JoinCondition, - pub left: Box, - pub right: Box, -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum JoinOperator { - Inner, - // Outer joins can not work with `JoinCondition::None` - LeftOuter, - RightOuter, - FullOuter, - LeftSemi, - LeftAnti, - RightSemi, - RightAnti, - // CrossJoin can only work with `JoinCondition::None` - CrossJoin, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum JoinCondition { - On(Box), - Using(Vec), - Natural, - None, -} - -impl SetExpr { - pub fn span(&self) -> Span { - match self { - SetExpr::Select(stmt) => stmt.span, - SetExpr::Query(query) => query.span, - SetExpr::SetOperation(op) => op.span, - SetExpr::Values { span, .. } => *span, - } - } - - pub fn into_query(self) -> Query { - match self { - SetExpr::Query(query) => *query, - _ => Query { - span: self.span(), - with: None, - body: self, - order_by: vec![], - limit: vec![], - offset: None, - ignore_result: false, - }, - } - } -} - -impl Display for OrderByExpr { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.expr)?; - if let Some(asc) = self.asc { - if asc { - write!(f, " ASC")?; - } else { - write!(f, " DESC")?; - } - } - if let Some(nulls_first) = self.nulls_first { - if nulls_first { - write!(f, " NULLS FIRST")?; - } else { - write!(f, " NULLS LAST")?; - } - } - Ok(()) - } -} - -impl Display for TableAlias { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", &self.name)?; - if !self.columns.is_empty() { - write!(f, "(")?; - write_comma_separated_list(f, &self.columns)?; - write!(f, ")")?; - } - Ok(()) - } -} - -impl Display for Pivot { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "PIVOT({} FOR {} IN (", self.aggregate, self.value_column)?; - write_comma_separated_list(f, &self.values)?; - write!(f, "))")?; - Ok(()) - } -} - -impl Display for Unpivot { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "UNPIVOT({} FOR {} IN (", - self.value_column, self.column_name - )?; - write_comma_separated_list(f, &self.names)?; - write!(f, "))")?; - Ok(()) - } -} - impl Display for TableReference { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { @@ -623,225 +771,77 @@ impl Display for TableReference { } } -impl Display for Indirection { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Indirection::Identifier(ident) => { - write!(f, "{ident}")?; - } - Indirection::Star(_) => { - write!(f, "*")?; - } - } - Ok(()) - } -} - -impl Display for SelectTarget { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - SelectTarget::AliasedExpr { expr, alias } => { - write!(f, "{expr}")?; - if let Some(ident) = alias { - write!(f, " AS {ident}")?; - } - } - SelectTarget::StarColumns { - qualified, - column_filter, - } => match column_filter { - Some(ColumnFilter::Excludes(excludes)) => { - write_dot_separated_list(f, qualified)?; - write!(f, " EXCLUDE (")?; - write_comma_separated_list(f, excludes)?; - write!(f, ")")?; - } - Some(ColumnFilter::Lambda(lambda)) => { - write!(f, "COLUMNS({lambda})")?; - } - None => { - write_dot_separated_list(f, qualified)?; - } - }, - } - Ok(()) - } +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub struct TableAlias { + pub name: Identifier, + pub columns: Vec, } -impl Display for SelectStmt { +impl Display for TableAlias { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // SELECT clause - write!(f, "SELECT ")?; - if let Some(hints) = &self.hints { - write!(f, "{} ", hints)?; - } - if self.distinct { - write!(f, "DISTINCT ")?; - } - write_comma_separated_list(f, &self.select_list)?; - - // FROM clause - if !self.from.is_empty() { - write!(f, " FROM ")?; - write_comma_separated_list(f, &self.from)?; - } - - // WHERE clause - if let Some(expr) = &self.selection { - write!(f, " WHERE {expr}")?; - } - - // GROUP BY clause - if self.group_by.is_some() { - write!(f, " GROUP BY ")?; - match self.group_by.as_ref().unwrap() { - GroupBy::Normal(exprs) => { - write_comma_separated_list(f, exprs)?; - } - GroupBy::All => { - write!(f, "ALL")?; - } - GroupBy::GroupingSets(sets) => { - write!(f, "GROUPING SETS (")?; - for (i, set) in sets.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "(")?; - write_comma_separated_list(f, set)?; - write!(f, ")")?; - } - write!(f, ")")?; - } - GroupBy::Cube(exprs) => { - write!(f, "CUBE (")?; - write_comma_separated_list(f, exprs)?; - write!(f, ")")?; - } - GroupBy::Rollup(exprs) => { - write!(f, "ROLLUP (")?; - write_comma_separated_list(f, exprs)?; - write!(f, ")")?; - } - } - } - - // HAVING clause - if let Some(having) = &self.having { - write!(f, " HAVING {having}")?; + write!(f, "{}", &self.name)?; + if !self.columns.is_empty() { + write!(f, "(")?; + write_comma_separated_list(f, &self.columns)?; + write!(f, ")")?; } - Ok(()) } } -impl Display for SetExpr { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - SetExpr::Select(select_stmt) => { - write!(f, "{select_stmt}")?; - } - SetExpr::Query(query) => { - write!(f, "({query})")?; - } - SetExpr::SetOperation(set_operation) => { - write!(f, "{}", set_operation.left)?; - match set_operation.op { - SetOperator::Union => { - write!(f, " UNION ")?; - } - SetOperator::Except => { - write!(f, " EXCEPT ")?; - } - SetOperator::Intersect => { - write!(f, " INTERSECT ")?; - } - } - if set_operation.all { - write!(f, "ALL ")?; - } - write!(f, "{}", set_operation.right)?; - } - SetExpr::Values { values, .. } => { - write!(f, "VALUES")?; - for (i, value) in values.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "(")?; - write_comma_separated_list(f, value)?; - write!(f, ")")?; - } - } - } - Ok(()) - } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct Join { + pub op: JoinOperator, + pub condition: JoinCondition, + pub left: Box, + pub right: Box, } -impl Display for CTE { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{} AS ", self.alias)?; - if self.materialized { - write!(f, "MATERIALIZED ")?; - } - write!(f, "({})", self.query)?; - Ok(()) - } +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum JoinOperator { + Inner, + // Outer joins can not work with `JoinCondition::None` + LeftOuter, + RightOuter, + FullOuter, + LeftSemi, + LeftAnti, + RightSemi, + RightAnti, + // CrossJoin can only work with `JoinCondition::None` + CrossJoin, } -impl Display for With { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - if self.recursive { - write!(f, "RECURSIVE ")?; - } - - write_comma_separated_list(f, &self.ctes)?; - Ok(()) - } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum JoinCondition { + On(Box), + Using(Vec), + Natural, + None, } -impl Display for Query { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // CTE, with clause - if let Some(with) = &self.with { - write!(f, "WITH {with} ")?; - } - - // Query body - write!(f, "{}", self.body)?; - - // ORDER BY clause - if !self.order_by.is_empty() { - write!(f, " ORDER BY ")?; - write_comma_separated_list(f, &self.order_by)?; - } - - // LIMIT clause - if !self.limit.is_empty() { - write!(f, " LIMIT ")?; - write_comma_separated_list(f, &self.limit)?; - } - - // TODO: We should validate if offset exists, limit should be empty or just one element - if let Some(offset) = &self.offset { - write!(f, " OFFSET {offset}")?; +impl SetExpr { + pub fn span(&self) -> Span { + match self { + SetExpr::Select(stmt) => stmt.span, + SetExpr::Query(query) => query.span, + SetExpr::SetOperation(op) => op.span, + SetExpr::Values { span, .. } => *span, } - - Ok(()) } -} -impl Display for TimeTravelPoint { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + pub fn into_query(self) -> Query { match self { - TimeTravelPoint::Snapshot(sid) => { - write!(f, " (SNAPSHOT => {sid})")?; - } - TimeTravelPoint::Timestamp(ts) => { - write!(f, " (TIMESTAMP => {ts})")?; - } + SetExpr::Query(query) => *query, + _ => Query { + span: self.span(), + with: None, + body: self, + order_by: vec![], + limit: vec![], + offset: None, + ignore_result: false, + }, } - - Ok(()) } } diff --git a/src/query/ast/src/ast/statements/catalog.rs b/src/query/ast/src/ast/statements/catalog.rs index a5aa9841a12ee..5bbed5c8e123e 100644 --- a/src/query/ast/src/ast/statements/catalog.rs +++ b/src/query/ast/src/ast/statements/catalog.rs @@ -20,7 +20,7 @@ use databend_common_meta_app::schema::CatalogType; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_map; +use crate::ast::write_comma_separated_string_map; use crate::ast::Identifier; use crate::ast::ShowLimit; @@ -70,9 +70,9 @@ impl Display for CreateCatalogStmt { write!(f, " IF NOT EXISTS")?; } write!(f, " {}", self.catalog_name)?; - write!(f, " TYPE='{}'", self.catalog_type)?; + write!(f, " TYPE={}", self.catalog_type)?; write!(f, " CONNECTION = ( ")?; - write_comma_separated_map(f, &self.catalog_options)?; + write_comma_separated_string_map(f, &self.catalog_options)?; write!(f, " )") } } diff --git a/src/query/ast/src/ast/statements/connection.rs b/src/query/ast/src/ast/statements/connection.rs index 8e568d8eda724..985a3e8fedbc8 100644 --- a/src/query/ast/src/ast/statements/connection.rs +++ b/src/query/ast/src/ast/statements/connection.rs @@ -16,7 +16,6 @@ use std::collections::BTreeMap; use std::fmt::Display; use std::fmt::Formatter; -use databend_common_base::base::mask_string; use databend_common_meta_app::schema::CreateOption; use derive_visitor::Drive; use derive_visitor::DriveMut; @@ -41,14 +40,21 @@ pub struct DropConnectionStmt { pub name: Identifier, } +impl Display for DropConnectionStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "DROP CONNECTION ")?; + if self.if_exists { + write!(f, "IF EXISTS ")?; + } + write!(f, "{} ", self.name) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub struct DescribeConnectionStmt { pub name: Identifier, } -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub struct ShowConnectionsStmt {} - impl Display for CreateConnectionStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "CREATE")?; @@ -60,30 +66,23 @@ impl Display for CreateConnectionStmt { write!(f, "IF NOT EXISTS ")?; } write!(f, "{} ", self.name)?; - write!(f, "STORAGE_TYPE = {} ", self.storage_type)?; + write!(f, "STORAGE_TYPE = '{}'", self.storage_type)?; for (k, v) in &self.storage_params { - write!(f, "{} = {}", k, mask_string(v, 3))?; + write!(f, " {k} = '{v}'")?; } Ok(()) } } -impl Display for DropConnectionStmt { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "CREATE CONNECTION ")?; - if self.if_exists { - write!(f, "IF NOT EXISTS ")?; - } - write!(f, "{} ", self.name) - } -} - impl Display for DescribeConnectionStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "CREATE CONNECTION {} ", self.name) + write!(f, "DESCRIBE CONNECTION {} ", self.name) } } +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub struct ShowConnectionsStmt {} + impl Display for ShowConnectionsStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "SHOW CONNECTIONS") diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index 392dbb448b36a..d4eb70975b5e3 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -24,6 +24,7 @@ use std::str::FromStr; use databend_common_base::base::mask_string; use databend_common_exception::ErrorCode; use databend_common_meta_app::principal::CopyOptions; +use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::OnErrorMode; use databend_common_meta_app::principal::COPY_MAX_FILES_PER_COMMIT; use derive_visitor::Drive; @@ -32,7 +33,8 @@ use itertools::Itertools; use url::Url; use crate::ast::write_comma_separated_map; -use crate::ast::write_comma_separated_quoted_list; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; @@ -53,8 +55,7 @@ pub struct CopyIntoTableStmt { pub hints: Option, - #[drive(skip)] - pub file_format: BTreeMap, + pub file_format: FileFormatOptions, // files to load #[drive(skip)] @@ -147,7 +148,7 @@ impl Display for CopyIntoTableStmt { if let Some(files) = &self.files { write!(f, " FILES = (")?; - write_comma_separated_quoted_list(f, files)?; + write_comma_separated_string_list(f, files)?; write!(f, " )")?; } @@ -156,9 +157,7 @@ impl Display for CopyIntoTableStmt { } if !self.file_format.is_empty() { - write!(f, " FILE_FORMAT = (")?; - write_comma_separated_map(f, &self.file_format)?; - write!(f, ")")?; + write!(f, " FILE_FORMAT = ({})", self.file_format)?; } if !self.validation_mode.is_empty() { @@ -193,7 +192,7 @@ pub struct CopyIntoLocationStmt { pub src: CopyIntoLocationSource, pub dst: FileLocation, #[drive(skip)] - pub file_format: BTreeMap, + pub file_format: FileFormatOptions, #[drive(skip)] pub single: bool, #[drive(skip)] @@ -212,9 +211,7 @@ impl Display for CopyIntoLocationStmt { write!(f, " FROM {}", self.src)?; if !self.file_format.is_empty() { - write!(f, " FILE_FORMAT = (")?; - write_comma_separated_map(f, &self.file_format)?; - write!(f, ")")?; + write!(f, " FILE_FORMAT = ({})", self.file_format)?; } write!(f, " SINGLE = {}", self.single)?; write!(f, " MAX_FILE_SIZE = {}", self.max_file_size)?; @@ -246,7 +243,7 @@ pub enum CopyIntoTableSource { impl Display for CopyIntoTableSource { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - CopyIntoTableSource::Location(v) => v.fmt(f), + CopyIntoTableSource::Location(location) => write!(f, "{location}"), CopyIntoTableSource::Query(query) => { write!(f, "({query})") } @@ -335,7 +332,7 @@ impl Display for Connection { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if !self.conns.is_empty() { write!(f, " CONNECTION = ( ")?; - write_comma_separated_map(f, &self.conns)?; + write_comma_separated_string_map(f, &self.conns)?; write!(f, " )")?; } Ok(()) @@ -433,10 +430,10 @@ impl UriLocation { impl Display for UriLocation { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "'{}://{}{}'", self.protocol, self.name, self.path)?; + write!(f, "{}", self.connection)?; if !self.part_prefix.is_empty() { write!(f, " LOCATION_PREFIX = '{}'", self.part_prefix)?; } - write!(f, "{}", self.connection.mask())?; Ok(()) } } @@ -464,7 +461,7 @@ impl Display for FileLocation { write!(f, "{}", loc) } FileLocation::Stage(loc) => { - write!(f, "@{}", loc) + write!(f, "'@{}'", loc) } } } @@ -473,7 +470,7 @@ impl Display for FileLocation { pub enum CopyIntoTableOption { Files(Vec), Pattern(String), - FileFormat(BTreeMap), + FileFormat(FileFormatOptions), ValidationMode(String), SizeLimit(usize), MaxFiles(usize), @@ -486,8 +483,65 @@ pub enum CopyIntoTableOption { } pub enum CopyIntoLocationOption { - FileFormat(BTreeMap), + FileFormat(FileFormatOptions), MaxFileSize(usize), Single(bool), DetailedOutput(bool), } + +#[derive(Clone, Debug, PartialEq, Eq, Default, Drive, DriveMut)] +pub struct FileFormatOptions { + #[drive(skip)] + pub options: BTreeMap, +} + +impl FileFormatOptions { + pub fn is_empty(&self) -> bool { + self.options.is_empty() + } + + pub fn to_meta_ast(&self) -> FileFormatOptionsAst { + let options = self + .options + .iter() + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(); + FileFormatOptionsAst { options } + } +} + +impl Display for FileFormatOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write_comma_separated_map(f, &self.options) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FileFormatValue { + Keyword(String), + Bool(bool), + U64(u64), + String(String), + StringList(Vec), +} + +impl Display for FileFormatValue { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + FileFormatValue::Keyword(v) => write!(f, "{v}"), + FileFormatValue::Bool(v) => write!(f, "{v}"), + FileFormatValue::U64(v) => write!(f, "{v}"), + FileFormatValue::String(v) => write!(f, "'{v}'"), + FileFormatValue::StringList(v) => { + write!(f, "(")?; + for (i, s) in v.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "'{s}'")?; + } + write!(f, ")") + } + } + } +} diff --git a/src/query/ast/src/ast/statements/database.rs b/src/query/ast/src/ast/statements/database.rs index 368705a782665..c3eae2ad180db 100644 --- a/src/query/ast/src/ast/statements/database.rs +++ b/src/query/ast/src/ast/statements/database.rs @@ -79,13 +79,13 @@ pub struct CreateDatabaseStmt { impl Display for CreateDatabaseStmt { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "CREATE")?; + write!(f, "CREATE ")?; if let CreateOption::CreateOrReplace = self.create_option { - write!(f, " OR REPLACE")?; + write!(f, "OR REPLACE ")?; } - write!(f, " DATABASE")?; + write!(f, "DATABASE ")?; if let CreateOption::CreateIfNotExists = self.create_option { - write!(f, " IF NOT EXISTS ")?; + write!(f, "IF NOT EXISTS ")?; } write!(f, "{}", self.database)?; diff --git a/src/query/ast/src/ast/statements/explain.rs b/src/query/ast/src/ast/statements/explain.rs index 6a99caacc3e9e..1777cd13e9cf9 100644 --- a/src/query/ast/src/ast/statements/explain.rs +++ b/src/query/ast/src/ast/statements/explain.rs @@ -41,7 +41,7 @@ pub enum ExplainKind { #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub enum ExplainOption { - Verbose(#[drive(skip)] bool), - Logical(#[drive(skip)] bool), - Optimized(#[drive(skip)] bool), + Verbose, + Logical, + Optimized, } diff --git a/src/query/ast/src/ast/statements/index.rs b/src/query/ast/src/ast/statements/index.rs index fa81a468ba997..f385ede6ac6c1 100644 --- a/src/query/ast/src/ast/statements/index.rs +++ b/src/query/ast/src/ast/statements/index.rs @@ -63,8 +63,10 @@ impl Display for CreateIndexStmt { if let CreateOption::CreateOrReplace = self.create_option { write!(f, "OR REPLACE ")?; } - let sync = if self.sync_creation { "SYNC" } else { "ASYNC" }; - write!(f, "{} {} INDEX", sync, self.index_type)?; + if !self.sync_creation { + write!(f, "ASYNC ")?; + } + write!(f, "{} INDEX", self.index_type)?; if let CreateOption::CreateIfNotExists = self.create_option { write!(f, " IF NOT EXISTS")?; } @@ -132,8 +134,10 @@ impl Display for CreateInvertedIndexStmt { if let CreateOption::CreateOrReplace = self.create_option { write!(f, "OR REPLACE ")?; } - let sync = if self.sync_creation { "SYNC" } else { "ASYNC" }; - write!(f, "{} INVERTED INDEX", sync)?; + if !self.sync_creation { + write!(f, "ASYNC ")?; + } + write!(f, "INVERTED INDEX")?; if let CreateOption::CreateIfNotExists = self.create_option { write!(f, " IF NOT EXISTS")?; } diff --git a/src/query/ast/src/ast/statements/insert.rs b/src/query/ast/src/ast/statements/insert.rs index 046c581c9c8f1..5a448ab0ff865 100644 --- a/src/query/ast/src/ast/statements/insert.rs +++ b/src/query/ast/src/ast/statements/insert.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; use std::fmt::Display; use std::fmt::Formatter; @@ -20,9 +19,9 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_map; use crate::ast::write_dot_separated_list; use crate::ast::Expr; +use crate::ast::FileFormatOptions; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; @@ -77,8 +76,7 @@ pub enum InsertSource { start: usize, }, StreamingV2 { - #[drive(skip)] - settings: BTreeMap, + settings: FileFormatOptions, #[drive(skip)] on_error_mode: Option, #[drive(skip)] @@ -111,9 +109,7 @@ impl Display for InsertSource { on_error_mode, start: _, } => { - write!(f, " FILE_FORMAT = (")?; - write_comma_separated_map(f, settings)?; - write!(f, " )")?; + write!(f, " FILE_FORMAT = ({})", settings)?; write!( f, " ON_ERROR = '{}'", diff --git a/src/query/ast/src/ast/statements/merge_into.rs b/src/query/ast/src/ast/statements/merge_into.rs index 91c15bdc87964..4de5854004a21 100644 --- a/src/query/ast/src/ast/statements/merge_into.rs +++ b/src/query/ast/src/ast/statements/merge_into.rs @@ -22,9 +22,9 @@ use derive_visitor::Drive; use derive_visitor::DriveMut; use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_map; use crate::ast::write_dot_separated_list; use crate::ast::Expr; +use crate::ast::FileFormatOptions; use crate::ast::Hint; use crate::ast::Identifier; use crate::ast::Query; @@ -217,8 +217,7 @@ impl MergeIntoStmt { #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum MergeSource { StreamingV2 { - #[drive(skip)] - settings: BTreeMap, + settings: FileFormatOptions, #[drive(skip)] on_error_mode: Option, #[drive(skip)] @@ -293,9 +292,7 @@ impl Display for MergeSource { on_error_mode, start: _, } => { - write!(f, " FILE_FORMAT = (")?; - write_comma_separated_map(f, settings)?; - write!(f, " )")?; + write!(f, " FILE_FORMAT = ({})", settings)?; write!( f, " ON_ERROR = '{}'", diff --git a/src/query/ast/src/ast/statements/notification.rs b/src/query/ast/src/ast/statements/notification.rs index 7844bb3524bb7..c10e7a7ea4118 100644 --- a/src/query/ast/src/ast/statements/notification.rs +++ b/src/query/ast/src/ast/statements/notification.rs @@ -30,7 +30,7 @@ pub struct CreateNotificationStmt { pub enabled: bool, pub webhook_opts: Option, #[drive(skip)] - pub comments: String, + pub comments: Option, } impl Display for CreateNotificationStmt { @@ -41,11 +41,12 @@ impl Display for CreateNotificationStmt { } write!(f, " {}", self.name)?; write!(f, " TYPE = {}", self.notification_type)?; + write!(f, " ENABLED = {}", self.enabled)?; if let Some(webhook_opts) = &self.webhook_opts { write!(f, " {}", webhook_opts)?; } - if !self.comments.is_empty() { - write!(f, " COMMENTS = '{}'", self.comments)?; + if let Some(comments) = &self.comments { + write!(f, " COMMENTS = '{comments}'")?; } Ok(()) } @@ -69,9 +70,9 @@ impl Display for NotificationWebhookOptions { authorization_header, } = self; { - write!(f, " WEBHOOK = (")?; + write!(f, "WEBHOOK = (")?; if let Some(url) = url { - write!(f, "URL = '{}'", url)?; + write!(f, " URL = '{}'", url)?; } if let Some(method) = method { write!(f, " METHOD = '{}'", method)?; diff --git a/src/query/ast/src/ast/statements/presign.rs b/src/query/ast/src/ast/statements/presign.rs index b7dc079c9e9b3..3cb377dafb88b 100644 --- a/src/query/ast/src/ast/statements/presign.rs +++ b/src/query/ast/src/ast/statements/presign.rs @@ -19,6 +19,8 @@ use std::time::Duration; use derive_visitor::Drive; use derive_visitor::DriveMut; +use crate::parser::unescape::escape_at_string; + #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub enum PresignAction { Download, @@ -49,7 +51,7 @@ pub enum PresignLocation { impl Display for PresignLocation { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - PresignLocation::StageLocation(v) => v.fmt(f), + PresignLocation::StageLocation(v) => write!(f, "@{}", escape_at_string(v)), } } } diff --git a/src/query/ast/src/ast/statements/share.rs b/src/query/ast/src/ast/statements/share.rs index c5662bc730e4e..97cce82e92d93 100644 --- a/src/query/ast/src/ast/statements/share.rs +++ b/src/query/ast/src/ast/statements/share.rs @@ -24,6 +24,7 @@ use derive_visitor::DriveMut; use itertools::Itertools; use super::UriLocation; +use crate::ast::write_comma_separated_string_map; use crate::ast::Identifier; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] @@ -52,9 +53,7 @@ impl Display for CreateShareEndpointStmt { write!(f, "{}", self.endpoint)?; write!(f, " URL={}", self.url)?; write!(f, " TENANT={} ARGS=(", self.tenant)?; - for (k, v) in self.args.iter() { - write!(f, "{}={},", k, v)?; - } + write_comma_separated_string_map(f, &self.args)?; write!(f, ")")?; if let Some(comment) = &self.comment { write!(f, " COMMENT = '{comment}'")?; diff --git a/src/query/ast/src/ast/statements/stage.rs b/src/query/ast/src/ast/statements/stage.rs index 4538da8c8ceb9..02f525d0085e9 100644 --- a/src/query/ast/src/ast/statements/stage.rs +++ b/src/query/ast/src/ast/statements/stage.rs @@ -21,8 +21,9 @@ use databend_common_meta_app::schema::CreateOption; use derive_visitor::Drive; use derive_visitor::DriveMut; -use crate::ast::write_comma_separated_map; -use crate::ast::write_comma_separated_quoted_list; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; +use crate::ast::FileFormatOptions; use crate::ast::UriLocation; #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] @@ -34,8 +35,7 @@ pub struct CreateStageStmt { pub location: Option, - #[drive(skip)] - pub file_format_options: BTreeMap, + pub file_format_options: FileFormatOptions, #[drive(skip)] pub on_error: String, #[drive(skip)] @@ -46,6 +46,46 @@ pub struct CreateStageStmt { pub comments: String, } +impl Display for CreateStageStmt { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "CREATE")?; + if let CreateOption::CreateOrReplace = self.create_option { + write!(f, " OR REPLACE")?; + } + write!(f, " STAGE")?; + if let CreateOption::CreateIfNotExists = self.create_option { + write!(f, " IF NOT EXISTS")?; + } + write!(f, " {}", self.stage_name)?; + + if let Some(ul) = &self.location { + write!(f, " {ul}")?; + } + + if !self.file_format_options.is_empty() { + write!(f, " FILE_FORMAT = ({})", self.file_format_options)?; + } + + if !self.on_error.is_empty() { + write!(f, " ON_ERROR = '{}'", self.on_error)?; + } + + if self.size_limit != 0 { + write!(f, " SIZE_LIMIT = {}", self.size_limit)?; + } + + if !self.validation_mode.is_empty() { + write!(f, " VALIDATION_MODE = {}", self.validation_mode)?; + } + + if !self.comments.is_empty() { + write!(f, " COMMENTS = '{}'", self.comments)?; + } + + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] pub enum SelectStageOption { Files(#[drive(skip)] Vec), @@ -90,48 +130,6 @@ impl SelectStageOptions { } } -impl Display for CreateStageStmt { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "CREATE")?; - if let CreateOption::CreateOrReplace = self.create_option { - write!(f, " OR REPLACE")?; - } - write!(f, " STAGE")?; - if let CreateOption::CreateIfNotExists = self.create_option { - write!(f, " IF NOT EXISTS")?; - } - write!(f, " {}", self.stage_name)?; - - if let Some(ul) = &self.location { - write!(f, " {ul}")?; - } - - if !self.file_format_options.is_empty() { - write!(f, " FILE_FORMAT = (")?; - write_comma_separated_map(f, &self.file_format_options)?; - write!(f, " )")?; - } - - if !self.on_error.is_empty() { - write!(f, " ON_ERROR = '{}'", self.on_error)?; - } - - if self.size_limit != 0 { - write!(f, " SIZE_LIMIT = {}", self.size_limit)?; - } - - if !self.validation_mode.is_empty() { - write!(f, " VALIDATION_MODE = {}", self.validation_mode)?; - } - - if !self.comments.is_empty() { - write!(f, " COMMENTS = '{}'", self.comments)?; - } - - Ok(()) - } -} - // SELECT FROM // {@[/] | ''} [( // [ PATTERN => ''] @@ -153,7 +151,7 @@ impl Display for SelectStageOptions { if let Some(files) = self.files.as_ref() { write!(f, " FILES => (")?; - write_comma_separated_quoted_list(f, files)?; + write_comma_separated_string_list(f, files)?; write!(f, "),")?; } @@ -167,7 +165,7 @@ impl Display for SelectStageOptions { if !self.connection.is_empty() { write!(f, " CONNECTION => (")?; - write_comma_separated_map(f, &self.connection)?; + write_comma_separated_string_map(f, &self.connection)?; write!(f, " )")?; } diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index b75246753f396..4952ededc175b 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -15,7 +15,6 @@ use std::fmt::Display; use std::fmt::Formatter; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::PrincipalIdentity; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::schema::CreateOption; @@ -254,8 +253,7 @@ pub enum Statement { create_option: CreateOption, #[drive(skip)] name: String, - #[drive(skip)] - file_format_options: FileFormatOptionsAst, + file_format_options: FileFormatOptions, }, DropFileFormat { #[drive(skip)] @@ -369,6 +367,16 @@ impl Statement { _ => format!("{}", self), } } + + pub fn assert_idempotent_parser(&self) { + dbg!(&self); + let sql = self.to_string(); + let tokens = crate::parser::tokenize_sql(&sql).unwrap(); + let (parsed, _) = + crate::parser::parse_sql(&tokens, crate::parser::Dialect::PostgreSQL).unwrap(); + let reparsed = parsed.to_string(); + assert_eq!(sql, reparsed); + } } impl Display for Statement { @@ -388,15 +396,9 @@ impl Display for Statement { .iter() .map(|opt| { match opt { - ExplainOption::Verbose(v) => { - format!("VERBOSE = {}", v) - } - ExplainOption::Logical(v) => { - format!("LOGICAL = {}", v) - } - ExplainOption::Optimized(v) => { - format!("OPTIMIZED = {}", v) - } + ExplainOption::Verbose => "VERBOSE", + ExplainOption::Logical => "LOGICAL", + ExplainOption::Optimized => "OPTIMIZED", } }) .join(", ") @@ -420,12 +422,12 @@ impl Display for Statement { Statement::ExplainAnalyze { query } => { write!(f, "EXPLAIN ANALYZE {query}")?; } - Statement::Query(query) => write!(f, "{query}")?, - Statement::Insert(insert) => write!(f, "{insert}")?, - Statement::Replace(replace) => write!(f, "{replace}")?, - Statement::MergeInto(merge_into) => write!(f, "{merge_into}")?, - Statement::Delete(delete) => write!(f, "{delete}")?, - Statement::Update(update) => write!(f, "{update}")?, + Statement::Query(stmt) => write!(f, "{stmt}")?, + Statement::Insert(stmt) => write!(f, "{stmt}")?, + Statement::Replace(stmt) => write!(f, "{stmt}")?, + Statement::MergeInto(stmt) => write!(f, "{stmt}")?, + Statement::Delete(stmt) => write!(f, "{stmt}")?, + Statement::Update(stmt) => write!(f, "{stmt}")?, Statement::CopyIntoTable(stmt) => write!(f, "{stmt}")?, Statement::CopyIntoLocation(stmt) => write!(f, "{stmt}")?, Statement::ShowSettings { show_options } => { @@ -499,7 +501,7 @@ impl Display for Statement { } write!(f, "{variable} = {value}")?; } - Statement::UnSetVariable(unset) => write!(f, "{unset}")?, + Statement::UnSetVariable(stmt) => write!(f, "{stmt}")?, Statement::SetRole { is_default, role_name, @@ -508,7 +510,7 @@ impl Display for Statement { if *is_default { write!(f, "DEFAULT")?; } else { - write!(f, "{role_name}")?; + write!(f, "'{role_name}'")?; } } Statement::SetSecondaryRoles { option } => { @@ -631,7 +633,7 @@ impl Display for Statement { if_exists, stage_name, } => { - write!(f, "DROP STAGES")?; + write!(f, "DROP STAGE")?; if *if_exists { write!(f, " IF EXISTS")?; } @@ -654,7 +656,7 @@ impl Display for Statement { if let CreateOption::CreateOrReplace = create_option { write!(f, " OR REPLACE")?; } - write!(f, " FILE_FORMAT")?; + write!(f, " FILE FORMAT")?; if let CreateOption::CreateIfNotExists = create_option { write!(f, " IF NOT EXISTS")?; } @@ -662,7 +664,7 @@ impl Display for Statement { write!(f, " {file_format_options}")?; } Statement::DropFileFormat { if_exists, name } => { - write!(f, "DROP FILE_FORMAT")?; + write!(f, "DROP FILE FORMAT")?; if *if_exists { write!(f, " IF EXISTS")?; } diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index 067764924e6dc..298ab465b2b0d 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use std::fmt::Display; use std::fmt::Formatter; -use std::format; use std::time::Duration; use databend_common_meta_app::schema::CreateOption; @@ -24,7 +23,7 @@ use derive_visitor::DriveMut; use crate::ast::statements::show::ShowLimit; use crate::ast::write_comma_separated_list; -use crate::ast::write_comma_separated_map; +use crate::ast::write_comma_separated_string_map; use crate::ast::write_dot_separated_list; use crate::ast::Expr; use crate::ast::Identifier; @@ -188,7 +187,7 @@ impl Display for CreateTableStmt { } // Format table options - write_comma_separated_map(f, &self.table_options)?; + write_comma_separated_string_map(f, &self.table_options)?; if let Some(as_query) = &self.as_query { write!(f, " AS {as_query}")?; } @@ -392,7 +391,7 @@ impl Display for AlterTableAction { match self { AlterTableAction::SetOptions { set_options } => { write!(f, "SET OPTIONS (")?; - write_comma_separated_map(f, set_options)?; + write_comma_separated_string_map(f, set_options)?; write!(f, ")")?; } AlterTableAction::RenameTable { new_table } => { @@ -414,8 +413,9 @@ impl Display for AlterTableAction { write!(f, "DROP COLUMN {column}")?; } AlterTableAction::AlterTableClusterKey { cluster_by } => { - write!(f, "CLUSTER BY ")?; + write!(f, "CLUSTER BY (")?; write_comma_separated_list(f, cluster_by)?; + write!(f, ")")?; } AlterTableAction::DropTableClusterKey => { write!(f, "DROP CLUSTER KEY")?; @@ -844,35 +844,8 @@ impl Display for ModifyColumnAction { ModifyColumnAction::UnsetMaskingPolicy(column) => { write!(f, "{} UNSET MASKING POLICY", column)? } - ModifyColumnAction::SetDataType(column_def_vec) => { - let ret = column_def_vec - .iter() - .enumerate() - .map(|(i, column_def)| { - let default_expr_str = match &column_def.expr { - Some(default_expr) => default_expr.to_string(), - None => "".to_string(), - }; - let comment = match &column_def.comment { - Some(comment) => format!(" COMMENT {}", comment), - None => "".to_string(), - }; - if i > 0 { - format!( - " COLUMN {} {}{}{}", - column_def.name, column_def.data_type, default_expr_str, comment - ) - } else { - format!( - "{} {}{}{}", - column_def.name, column_def.data_type, default_expr_str, comment - ) - } - }) - .collect::>() - .join(","); - - write!(f, "{}", ret)? + ModifyColumnAction::SetDataType(column_defs) => { + write_comma_separated_list(f, column_defs)? } ModifyColumnAction::ConvertStoredComputedColumn(column) => { write!(f, "{} DROP STORED", column)? diff --git a/src/query/ast/src/ast/statements/task.rs b/src/query/ast/src/ast/statements/task.rs index 1ab544a505503..1542d2a9f1740 100644 --- a/src/query/ast/src/ast/statements/task.rs +++ b/src/query/ast/src/ast/statements/task.rs @@ -19,6 +19,9 @@ use std::fmt::Formatter; use derive_visitor::Drive; use derive_visitor::DriveMut; +use crate::ast::write_comma_separated_string_list; +use crate::ast::write_comma_separated_string_map; +use crate::ast::Expr; use crate::ast::ShowLimit; #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] @@ -59,11 +62,11 @@ pub struct CreateTaskStmt { #[drive(skip)] pub error_integration: Option, #[drive(skip)] - pub comments: String, + pub comments: Option, #[drive(skip)] pub after: Vec, #[drive(skip)] - pub when_condition: Option, + pub when_condition: Option, #[drive(skip)] pub sql: TaskSql, } @@ -74,43 +77,39 @@ impl Display for CreateTaskStmt { if self.if_not_exists { write!(f, " IF NOT EXISTS")?; } + write!(f, " {}", self.name)?; - write!(f, "{}", self.warehouse_opts)?; - if let Some(schedule_opt) = self.schedule_opts.as_ref() { - write!(f, "{}", schedule_opt)?; - } + write!(f, " {}", self.warehouse_opts)?; - if !self.session_parameters.is_empty() { - for (key, value) in &self.session_parameters { - write!(f, " {} = '{}'", key, value)?; - } + if let Some(schedule_opt) = self.schedule_opts.as_ref() { + write!(f, " SCHEDULE = {}", schedule_opt)?; } if let Some(num) = self.suspend_task_after_num_failures { - write!(f, " SUSPEND_TASK_AFTER {} FAILURES", num)?; - } - - if !self.comments.is_empty() { - write!(f, " COMMENTS = '{}'", self.comments)?; + write!(f, " SUSPEND_TASK_AFTER_NUM_FAILURES = {}", num)?; } if !self.after.is_empty() { - write!(f, " AFTER = '{:?}'", self.after)?; + write!(f, " AFTER ")?; + write_comma_separated_string_list(f, &self.after)?; } - if self.when_condition.is_some() { - write!(f, " WHEN = '{:?}'", self.when_condition)?; + if let Some(when_condition) = &self.when_condition { + write!(f, " WHEN {}", when_condition)?; } - if self.error_integration.is_some() { - write!( - f, - " ERROR INTEGRATION = '{}'", - self.error_integration.as_ref().unwrap() - )?; + if let Some(error_integration) = &self.error_integration { + write!(f, " ERROR_INTEGRATION = '{}'", error_integration)?; } + if let Some(comments) = &self.comments { + write!(f, " COMMENTS = '{}'", comments)?; + } + + write_comma_separated_string_map(f, &self.session_parameters)?; + write!(f, " AS {}", self.sql)?; + Ok(()) } } @@ -124,7 +123,7 @@ pub struct WarehouseOptions { impl Display for WarehouseOptions { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(wh) = &self.warehouse { - write!(f, " WAREHOUSE = {}", wh)?; + write!(f, "WAREHOUSE = '{}'", wh)?; } Ok(()) } @@ -140,12 +139,12 @@ impl Display for ScheduleOptions { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { ScheduleOptions::IntervalSecs(secs) => { - write!(f, " SCHEDULE {} SECOND", secs) + write!(f, "{} SECOND", secs) } ScheduleOptions::CronExpression(expr, tz) => { - write!(f, " SCHEDULE CRON '{}'", expr)?; + write!(f, "USING CRON '{}'", expr)?; if let Some(tz) = tz { - write!(f, " TIMEZONE '{}'", tz)?; + write!(f, " '{}'", tz)?; } Ok(()) } @@ -162,6 +161,18 @@ pub struct AlterTaskStmt { pub options: AlterTaskOptions, } +impl Display for AlterTaskStmt { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "ALTER TASK")?; + if self.if_exists { + write!(f, " IF EXISTS")?; + } + write!(f, " {}", self.name)?; + write!(f, " {}", self.options)?; + Ok(()) + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut)] pub enum AlterTaskOptions { Resume, @@ -185,7 +196,7 @@ pub enum AlterTaskOptions { }, // Change SQL ModifyAs(#[drive(skip)] TaskSql), - ModifyWhen(#[drive(skip)] String), + ModifyWhen(Expr), AddAfter(#[drive(skip)] Vec), RemoveAfter(#[drive(skip)] Vec), } @@ -193,8 +204,8 @@ pub enum AlterTaskOptions { impl Display for AlterTaskOptions { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - AlterTaskOptions::Resume => write!(f, " RESUME"), - AlterTaskOptions::Suspend => write!(f, " SUSPEND"), + AlterTaskOptions::Resume => write!(f, "RESUME"), + AlterTaskOptions::Suspend => write!(f, "SUSPEND"), AlterTaskOptions::Set { warehouse, schedule, @@ -203,51 +214,45 @@ impl Display for AlterTaskOptions { error_integration, comments, } => { + write!(f, "SET")?; if let Some(wh) = warehouse { - write!(f, " SET WAREHOUSE = {}", wh)?; + write!(f, " WAREHOUSE = '{wh}'")?; } if let Some(schedule) = schedule { - write!(f, " SET {}", schedule)?; - } - if let Some(error_integration) = error_integration { - write!(f, " ERROR INTEGRATION = '{}'", error_integration)?; + write!(f, " SCHEDULE = {schedule}")?; } if let Some(num) = suspend_task_after_num_failures { - write!(f, " SUSPEND TASK AFTER {} FAILURES", num)?; + write!(f, " SUSPEND_TASK_AFTER_NUM_FAILURES = {num}")?; } if let Some(comments) = comments { - write!(f, " COMMENTS = '{}'", comments)?; + write!(f, " COMMENT = '{comments}'")?; + } + if let Some(error_integration) = error_integration { + write!(f, " ERROR_INTEGRATION = '{error_integration}'")?; } if let Some(session) = session_parameters { - for (key, value) in session { - write!(f, " {} = '{}'", key, value)?; - } + write!(f, " ")?; + write_comma_separated_string_map(f, session)?; } Ok(()) } AlterTaskOptions::Unset { warehouse } => { if *warehouse { - write!(f, " UNSET WAREHOUSE")?; + write!(f, "UNSET WAREHOUSE")?; } Ok(()) } - AlterTaskOptions::ModifyAs(sql) => write!(f, " AS {}", sql), - AlterTaskOptions::ModifyWhen(when) => write!(f, " WHEN {}", when), - AlterTaskOptions::AddAfter(after) => write!(f, " ADD AFTER = '{:?}'", after), - AlterTaskOptions::RemoveAfter(after) => write!(f, " REMOVE AFTER = '{:?}'", after), - } - } -} - -impl Display for AlterTaskStmt { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "ALTER TASK")?; - if self.if_exists { - write!(f, " IF EXISTS")?; + AlterTaskOptions::ModifyAs(sql) => write!(f, "MODIFY AS {sql}"), + AlterTaskOptions::ModifyWhen(expr) => write!(f, "MODIFY WHEN {expr}"), + AlterTaskOptions::AddAfter(after) => { + write!(f, "ADD AFTER ")?; + write_comma_separated_string_list(f, after) + } + AlterTaskOptions::RemoveAfter(after) => { + write!(f, "REMOVE AFTER ")?; + write_comma_separated_string_list(f, after) + } } - write!(f, " {}", self.name)?; - write!(f, "{}", self.options)?; - Ok(()) } } diff --git a/src/query/ast/src/ast/statements/udf.rs b/src/query/ast/src/ast/statements/udf.rs index d6ebeeaeef8e2..656cb8fb337eb 100644 --- a/src/query/ast/src/ast/statements/udf.rs +++ b/src/query/ast/src/ast/statements/udf.rs @@ -55,24 +55,6 @@ pub enum UDFDefinition { }, } -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct CreateUDFStmt { - #[drive(skip)] - pub create_option: CreateOption, - pub udf_name: Identifier, - #[drive(skip)] - pub description: Option, - pub definition: UDFDefinition, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct AlterUDFStmt { - pub udf_name: Identifier, - #[drive(skip)] - pub description: Option, - pub definition: UDFDefinition, -} - impl Display for UDFDefinition { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { match self { @@ -95,7 +77,7 @@ impl Display for UDFDefinition { write_comma_separated_list(f, arg_types)?; write!( f, - ") RETURNS {return_type} LANGUAGE {language} HANDLER = {handler} ADDRESS = {address}" + ") RETURNS {return_type} LANGUAGE {language} HANDLER = '{handler}' ADDRESS = '{address}'" )?; } UDFDefinition::UDFScript { @@ -104,13 +86,13 @@ impl Display for UDFDefinition { code, handler, language, - runtime_version, + runtime_version: _, } => { write!(f, "(")?; write_comma_separated_list(f, arg_types)?; write!( f, - ") RETURNS {return_type} LANGUAGE {language} runtime_version = {runtime_version} HANDLER = {handler} AS $${code}$$" + ") RETURNS {return_type} LANGUAGE {language} HANDLER = '{handler}' AS $${code}$$" )?; } } @@ -118,6 +100,16 @@ impl Display for UDFDefinition { } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct CreateUDFStmt { + #[drive(skip)] + pub create_option: CreateOption, + pub udf_name: Identifier, + #[drive(skip)] + pub description: Option, + pub definition: UDFDefinition, +} + impl Display for CreateUDFStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "CREATE")?; @@ -136,6 +128,14 @@ impl Display for CreateUDFStmt { } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct AlterUDFStmt { + pub udf_name: Identifier, + #[drive(skip)] + pub description: Option, + pub definition: UDFDefinition, +} + impl Display for AlterUDFStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "ALTER FUNCTION")?; diff --git a/src/query/ast/src/ast/statements/update.rs b/src/query/ast/src/ast/statements/update.rs index 755f0ff35693c..4a8f3dcc48271 100644 --- a/src/query/ast/src/ast/statements/update.rs +++ b/src/query/ast/src/ast/statements/update.rs @@ -32,12 +32,6 @@ pub struct UpdateStmt { pub selection: Option, } -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct UpdateExpr { - pub name: Identifier, - pub expr: Expr, -} - impl Display for UpdateStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "UPDATE ")?; @@ -53,6 +47,12 @@ impl Display for UpdateStmt { } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct UpdateExpr { + pub name: Identifier, + pub expr: Expr, +} + impl Display for UpdateExpr { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "{} = {}", self.name, self.expr) diff --git a/src/query/ast/src/ast/statements/user.rs b/src/query/ast/src/ast/statements/user.rs index 3f76fd0f1ac22..0ad0fbf04cc24 100644 --- a/src/query/ast/src/ast/statements/user.rs +++ b/src/query/ast/src/ast/statements/user.rs @@ -50,10 +50,8 @@ impl Display for CreateUserStmt { write!(f, " {} IDENTIFIED", self.user)?; write!(f, " {}", self.auth_option)?; if !self.user_options.is_empty() { - write!(f, " WITH")?; - for user_option in &self.user_options { - write!(f, " {user_option}")?; - } + write!(f, " WITH ")?; + write_comma_separated_list(f, &self.user_options)?; } Ok(()) @@ -103,10 +101,8 @@ impl Display for AlterUserStmt { write!(f, " IDENTIFIED {}", auth_option)?; } if !self.user_options.is_empty() { - write!(f, " WITH")?; - for with_option in &self.user_options { - write!(f, " {with_option}")?; - } + write!(f, " WITH ")?; + write_comma_separated_list(f, &self.user_options)?; } Ok(()) diff --git a/src/query/ast/src/ast/visitors/visitor.rs b/src/query/ast/src/ast/visitors/visitor.rs index 1ba33dda0e644..96ce4713085c7 100644 --- a/src/query/ast/src/ast/visitors/visitor.rs +++ b/src/query/ast/src/ast/visitors/visitor.rs @@ -13,7 +13,6 @@ // limitations under the License. use databend_common_exception::Span; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::PrincipalIdentity; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::schema::CreateOption; @@ -648,7 +647,7 @@ pub trait Visitor<'ast>: Sized { &mut self, _create_option: &CreateOption, _name: &'ast str, - _file_format_options: &'ast FileFormatOptionsAst, + _file_format_options: &'ast FileFormatOptions, ) { } diff --git a/src/query/ast/src/ast/visitors/visitor_mut.rs b/src/query/ast/src/ast/visitors/visitor_mut.rs index f0d2fad07126b..a15d17ac98f82 100644 --- a/src/query/ast/src/ast/visitors/visitor_mut.rs +++ b/src/query/ast/src/ast/visitors/visitor_mut.rs @@ -13,7 +13,6 @@ // limitations under the License. use databend_common_exception::Span; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::PrincipalIdentity; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::schema::CreateOption; @@ -661,7 +660,7 @@ pub trait VisitorMut: Sized { &mut self, _create_option: &CreateOption, _name: &mut String, - _file_format_options: &mut FileFormatOptionsAst, + _file_format_options: &mut FileFormatOptions, ) { } diff --git a/src/query/ast/src/parser/stage.rs b/src/query/ast/src/parser/stage.rs index edf71b1cf519a..019833b2c0430 100644 --- a/src/query/ast/src/parser/stage.rs +++ b/src/query/ast/src/parser/stage.rs @@ -17,6 +17,8 @@ use std::collections::BTreeMap; use nom::branch::alt; use nom::combinator::map; +use crate::ast::FileFormatOptions; +use crate::ast::FileFormatValue; use crate::ast::FileLocation; use crate::ast::SelectStageOption; use crate::ast::UriLocation; @@ -72,24 +74,39 @@ pub fn connection_options(i: Input) -> IResult> { )(i) } -pub fn format_options(i: Input) -> IResult> { +pub fn format_options(i: Input) -> IResult { let option_type = map( rule! { TYPE ~ "=" ~ ( TSV | CSV | NDJSON | PARQUET | JSON | XML ) }, - |(_, _, v)| ("type".to_string(), v.text().to_string()), + |(_, _, v)| { + ( + "type".to_string(), + FileFormatValue::Keyword(v.text().to_string()), + ) + }, ); let option_compression = map( rule! { COMPRESSION ~ "=" ~ ( AUTO | NONE | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAWDEFLATE | XZ ) }, - |(_, _, v)| ("COMPRESSION".to_string(), v.text().to_string()), + |(_, _, v)| { + ( + "COMPRESSION".to_string(), + FileFormatValue::Keyword(v.text().to_string()), + ) + }, ); let ident_options = map( rule! { (BINARY_FORMAT | MISSING_FIELD_AS | EMPTY_FIELD_AS | NULL_FIELD_AS) ~ "=" ~ (NULL | STRING | Ident)}, - |(k, _, v)| (k.text().to_string(), v.text().to_string()), + |(k, _, v)| { + ( + k.text().to_string(), + FileFormatValue::Keyword(v.text().to_string()), + ) + }, ); let string_options = map( @@ -107,21 +124,21 @@ pub fn format_options(i: Input) -> IResult> { | MISSING_FIELD_AS | ROW_TAG) ~ ^"=" ~ ^#literal_string }, - |(k, _, v)| (k.text().to_string(), v), + |(k, _, v)| (k.text().to_string(), FileFormatValue::String(v)), ); let int_options = map( rule! { SKIP_HEADER ~ ^"=" ~ ^#literal_u64 }, - |(k, _, v)| (k.text().to_string(), v.to_string()), + |(k, _, v)| (k.text().to_string(), FileFormatValue::U64(v)), ); let bool_options = map( rule! { (ERROR_ON_COLUMN_COUNT_MISMATCH | OUTPUT_HEADER) ~ ^"=" ~ ^#literal_bool }, - |(k, _, v)| (k.text().to_string(), v.to_string()), + |(k, _, v)| (k.text().to_string(), FileFormatValue::Bool(v)), ); let none_options = map( @@ -133,17 +150,17 @@ pub fn format_options(i: Input) -> IResult> { | NON_DISPLAY | ESCAPE ) ~ ^"=" ~ ^NONE }, - |(k, _, v)| (k.text().to_string(), v.text().to_string()), + |(k, _, v)| { + ( + k.text().to_string(), + FileFormatValue::Keyword(v.text().to_string()), + ) + }, ); let null_if = map( rule! { NULL_IF ~ ^"=" ~ ^"(" ~ ^#comma_separated_list0(literal_string) ~ ^")" }, - |(_, _, _, values, _)| { - ( - "null_if".to_string(), - serde_json::to_string(&values).unwrap(), - ) - }, + |(_, _, _, values, _)| ("null_if".to_string(), FileFormatValue::StringList(values)), ); map( @@ -157,11 +174,16 @@ pub fn format_options(i: Input) -> IResult> { | #none_options | #null_if ) ~ ","?)* }, - |opts| BTreeMap::from_iter(opts.iter().map(|((k, v), _)| (k.to_lowercase(), v.clone()))), + |opts| FileFormatOptions { + options: opts + .iter() + .map(|((k, v), _)| (k.to_lowercase(), v.clone())) + .collect(), + }, )(i) } -pub fn file_format_clause(i: Input) -> IResult> { +pub fn file_format_clause(i: Input) -> IResult { map( rule! { FILE_FORMAT ~ ^"=" ~ ^"(" ~ ^#format_options ~ ^")" }, |(_, _, _, opts, _)| opts, diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index f116cc412dba6..de4d7bb9f4bb9 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -16,7 +16,6 @@ use std::collections::BTreeMap; use std::time::Duration; use databend_common_meta_app::principal::AuthType; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::PrincipalIdentity; use databend_common_meta_app::principal::UserIdentity; use databend_common_meta_app::principal::UserPrivilegeType; @@ -111,13 +110,13 @@ pub fn statement_body(i: Input) -> IResult { CREATE ~ TASK ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ #ident ~ #task_warehouse_option - ~ (SCHEDULE ~ "=" ~ #task_schedule_option)? - ~ (AFTER ~ #comma_separated_list0(literal_string))? - ~ (WHEN ~ #expr )? - ~ (SUSPEND_TASK_AFTER_NUM_FAILURES ~ "=" ~ #literal_u64)? + ~ ( SCHEDULE ~ "=" ~ #task_schedule_option )? + ~ ( AFTER ~ #comma_separated_list0(literal_string) )? + ~ ( WHEN ~ #expr )? + ~ ( SUSPEND_TASK_AFTER_NUM_FAILURES ~ "=" ~ #literal_u64 )? ~ ( ERROR_INTEGRATION ~ ^"=" ~ ^#literal_string )? ~ ( (COMMENT | COMMENTS) ~ ^"=" ~ ^#literal_string )? - ~ (#set_table_option)? + ~ #set_table_option? ~ AS ~ #task_sql_block }, |( @@ -143,13 +142,13 @@ pub fn statement_body(i: Input) -> IResult { warehouse_opts, schedule_opts: schedule_opts.map(|(_, _, opt)| opt), suspend_task_after_num_failures: suspend_opt.map(|(_, _, num)| num), - comments: comment_opt.map(|v| v.2).unwrap_or_default(), + comments: comment_opt.map(|(_, _, comment)| comment), after: match after_tasks { Some((_, tasks)) => tasks, None => Vec::new(), }, error_integration: error_integration.map(|(_, _, name)| name.to_string()), - when_condition: when_conditions.map(|(_, cond)| cond.to_string()), + when_condition: when_conditions.map(|(_, cond)| cond), sql, session_parameters: session_opts, }) @@ -1655,8 +1654,7 @@ pub fn statement_body(i: Input) -> IResult { CREATE ~ ( OR ~ ^REPLACE )? ~ FILE ~ FORMAT ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ #ident ~ #format_options }, - |(_, opt_or_replace, _, _, opt_if_not_exists, name, options)| { - let file_format_options = FileFormatOptionsAst { options }; + |(_, opt_or_replace, _, _, opt_if_not_exists, name, file_format_options)| { let create_option = parse_create_option(opt_or_replace.is_some(), opt_if_not_exists.is_some())?; Ok(Statement::CreateFileFormat { @@ -1945,11 +1943,12 @@ pub fn statement_body(i: Input) -> IResult { ); let create_notification = map( rule! { - CREATE ~ NOTIFICATION ~ INTEGRATION ~ ( IF ~ ^NOT ~ ^EXISTS )? + CREATE ~ NOTIFICATION ~ INTEGRATION + ~ ( IF ~ ^NOT ~ ^EXISTS )? ~ #ident ~ TYPE ~ "=" ~ #ident ~ ENABLED ~ "=" ~ #literal_bool - ~ (#notification_webhook_clause)? + ~ #notification_webhook_clause? ~ ( (COMMENT | COMMENTS) ~ ^"=" ~ ^#literal_string )? }, |( @@ -1973,7 +1972,7 @@ pub fn statement_body(i: Input) -> IResult { notification_type: notification_type.to_string(), enabled, webhook_opts: webhook, - comments: comment.map(|v| v.2).unwrap_or_default(), + comments: comment.map(|(_, _, comments)| comments), }) }, ); @@ -3362,10 +3361,7 @@ pub fn alter_task_option(i: Input) -> IResult { rule! { MODIFY ~ WHEN ~ #expr }, - |(_, _, expr)| { - let when = expr.to_string(); - AlterTaskOptions::ModifyWhen(when) - }, + |(_, _, expr)| AlterTaskOptions::ModifyWhen(expr), ); let add_after = map( rule! { @@ -3383,11 +3379,11 @@ pub fn alter_task_option(i: Input) -> IResult { let set = map( rule! { SET - ~ ( WAREHOUSE ~ "=" ~ #literal_string )? - ~ ( SCHEDULE ~ "=" ~ #task_schedule_option )? - ~ ( SUSPEND_TASK_AFTER_NUM_FAILURES ~ "=" ~ #literal_u64 )? - ~ ( COMMENT ~ "=" ~ #literal_string )? - ~ ( ERROR_INTEGRATION ~ "=" ~ #literal_string )? + ~ ( WAREHOUSE ~ ^"=" ~ ^#literal_string )? + ~ ( SCHEDULE ~ ^"=" ~ ^#task_schedule_option )? + ~ ( SUSPEND_TASK_AFTER_NUM_FAILURES ~ ^"=" ~ ^#literal_u64 )? + ~ ( COMMENT ~ ^"=" ~ ^#literal_string )? + ~ ( ERROR_INTEGRATION ~ ^"=" ~ ^#literal_string )? ~ #set_table_option? }, |( @@ -3459,7 +3455,7 @@ pub fn alter_pipe_option(i: Input) -> IResult { pub fn task_warehouse_option(i: Input) -> IResult { alt((map( rule! { - (WAREHOUSE ~ "=" ~ #literal_string)? + (WAREHOUSE ~ "=" ~ #literal_string)? }, |warehouse_opt| { let warehouse = match warehouse_opt { @@ -3558,19 +3554,17 @@ pub fn table_option(i: Input) -> IResult> { } pub fn set_table_option(i: Input) -> IResult> { - map( + let option = map( rule! { - ( #ident ~ "=" ~ #option_to_string ) ~ ("," ~ #ident ~ "=" ~ #option_to_string )* + #ident ~ "=" ~ #option_to_string }, - |(key, _, value, opts)| { - let mut options = BTreeMap::from_iter( - opts.iter() - .map(|(_, k, _, v)| (k.name.to_lowercase(), v.clone())), - ); - options.insert(key.name.to_lowercase(), value); - options - }, - )(i) + |(k, _, v)| (k, v), + ); + map(comma_separated_list1(option), |opts| { + opts.into_iter() + .map(|(k, v)| (k.name.to_lowercase(), v.clone())) + .collect() + })(i) } fn option_to_string(i: Input) -> IResult { @@ -3984,9 +3978,9 @@ pub fn explain_option(i: Input) -> IResult { VERBOSE | LOGICAL | OPTIMIZED }, |opt| match &opt.kind { - VERBOSE => ExplainOption::Verbose(true), - LOGICAL => ExplainOption::Logical(true), - OPTIMIZED => ExplainOption::Optimized(true), + VERBOSE => ExplainOption::Verbose, + LOGICAL => ExplainOption::Logical, + OPTIMIZED => ExplainOption::Optimized, _ => unreachable!(), }, )(i) diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index 48d20d322597a..d4e1e9953e84d 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -658,6 +658,7 @@ $$;"#, for case in cases { let tokens = tokenize_sql(case).unwrap(); let (stmt, fmt) = parse_sql(&tokens, Dialect::PostgreSQL).unwrap(); + stmt.assert_idempotent_parser(); writeln!(file, "---------- Input ----------").unwrap(); writeln!(file, "{}", case).unwrap(); writeln!(file, "---------- Output ---------").unwrap(); diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index 8a96180ea751b..c423e0dd655fb 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -682,20 +682,14 @@ Explain { ---------- Input ---------- explain(verbose, logical, optimized) select * from t where a = 1 ---------- Output --------- -EXPLAIN(VERBOSE = true, LOGICAL = true, OPTIMIZED = true) SELECT * FROM t WHERE (a = 1) +EXPLAIN(VERBOSE, LOGICAL, OPTIMIZED) SELECT * FROM t WHERE (a = 1) ---------- AST ------------ Explain { kind: Plan, options: [ - Verbose( - true, - ), - Logical( - true, - ), - Optimized( - true, - ), + Verbose, + Logical, + Optimized, ], query: Query( Query { @@ -843,7 +837,7 @@ Some( ---------- Input ---------- CREATE AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE b > 3 GROUP BY b; ---------- Output --------- -CREATE SYNC AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE (b > 3) GROUP BY b +CREATE AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE (b > 3) GROUP BY b ---------- AST ------------ CreateIndex( CreateIndexStmt { @@ -1036,7 +1030,7 @@ CreateIndex( ---------- Input ---------- CREATE OR REPLACE AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE b > 3 GROUP BY b; ---------- Output --------- -CREATE OR REPLACE SYNC AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE (b > 3) GROUP BY b +CREATE OR REPLACE AGGREGATING INDEX idx1 AS SELECT SUM(a), b FROM t1 WHERE (b > 3) GROUP BY b ---------- AST ------------ CreateIndex( CreateIndexStmt { @@ -1229,7 +1223,7 @@ CreateIndex( ---------- Input ---------- CREATE OR REPLACE INVERTED INDEX idx2 ON t1 (a, b); ---------- Output --------- -CREATE OR REPLACE SYNC INVERTED INDEX idx2 ON t1 (a, b) +CREATE OR REPLACE INVERTED INDEX idx2 ON t1 (a, b) ---------- AST ------------ CreateInvertedIndex( CreateInvertedIndexStmt { @@ -2169,7 +2163,7 @@ CreateTable( ---------- Input ---------- create table if not exists a.b (a int) 's3://testbucket/admin/data/' connection=(aws_key_id='minioadmin' aws_secret_key='minioadmin' endpoint_url='http://127.0.0.1:9900'); ---------- Output --------- -CREATE TABLE IF NOT EXISTS a.b (a Int32) 's3://testbucket/admin/data/' CONNECTION = ( aws_key_id = '******min', aws_secret_key = '******min', endpoint_url = '******900' ) +CREATE TABLE IF NOT EXISTS a.b (a Int32) 's3://testbucket/admin/data/' CONNECTION = ( aws_key_id = 'minioadmin', aws_secret_key = 'minioadmin', endpoint_url = 'http://127.0.0.1:9900' ) ---------- AST ------------ CreateTable( CreateTableStmt { @@ -2242,7 +2236,7 @@ create table if not exists a.b (a int) 's3://testbucket/admin/data/' connection=(aws_key_id='minioadmin' aws_secret_key='minioadmin' endpoint_url='http://127.0.0.1:9900') location_prefix = 'db'; ---------- Output --------- -CREATE TABLE IF NOT EXISTS a.b (a Int32) 's3://testbucket/admin/data/' LOCATION_PREFIX = 'db' CONNECTION = ( aws_key_id = '******min', aws_secret_key = '******min', endpoint_url = '******900' ) +CREATE TABLE IF NOT EXISTS a.b (a Int32) 's3://testbucket/admin/data/' CONNECTION = ( aws_key_id = 'minioadmin', aws_secret_key = 'minioadmin', endpoint_url = 'http://127.0.0.1:9900' ) LOCATION_PREFIX = 'db' ---------- AST ------------ CreateTable( CreateTableStmt { @@ -2442,7 +2436,7 @@ UseDatabase { ---------- Input ---------- create catalog ctl type=hive connection=(url='' thrift_protocol='binary'); ---------- Output --------- -CREATE CATALOG ctl TYPE='HIVE' CONNECTION = ( thrift_protocol = 'binary', url = '' ) +CREATE CATALOG ctl TYPE=HIVE CONNECTION = ( thrift_protocol = 'binary', url = '' ) ---------- AST ------------ CreateCatalog( CreateCatalogStmt { @@ -2486,7 +2480,7 @@ CreateDatabase( ---------- Input ---------- create database ctl.t engine = Default; ---------- Output --------- -CREATE DATABASEctl.t ENGINE = DEFAULT +CREATE DATABASE ctl.t ENGINE = DEFAULT ---------- AST ------------ CreateDatabase( CreateDatabaseStmt { @@ -2523,7 +2517,7 @@ CreateDatabase( ---------- Input ---------- create database t engine = Default; ---------- Output --------- -CREATE DATABASEt ENGINE = DEFAULT +CREATE DATABASE t ENGINE = DEFAULT ---------- AST ------------ CreateDatabase( CreateDatabaseStmt { @@ -2551,7 +2545,7 @@ CreateDatabase( ---------- Input ---------- create database t FROM SHARE a.s; ---------- Output --------- -CREATE DATABASEt FROM SHARE a.s +CREATE DATABASE t FROM SHARE a.s ---------- AST ------------ CreateDatabase( CreateDatabaseStmt { @@ -2582,7 +2576,7 @@ CreateDatabase( ---------- Input ---------- create or replace database a; ---------- Output --------- -CREATE OR REPLACE DATABASEa +CREATE OR REPLACE DATABASE a ---------- AST ------------ CreateDatabase( CreateDatabaseStmt { @@ -4335,7 +4329,7 @@ AlterUser( ---------- Input ---------- ALTER USER u1 WITH DEFAULT_ROLE = role1, TENANTSETTING; ---------- Output --------- -ALTER USER 'u1'@'%' WITH DEFAULT_ROLE = 'role1' TENANTSETTING +ALTER USER 'u1'@'%' WITH DEFAULT_ROLE = 'role1', TENANTSETTING ---------- AST ------------ AlterUser( AlterUserStmt { @@ -4405,7 +4399,7 @@ AlterUser( ---------- Input ---------- CREATE USER u1 IDENTIFIED BY '123456' WITH DEFAULT_ROLE='role123', TENANTSETTING ---------- Output --------- -CREATE USER 'u1'@'%' IDENTIFIED BY '123456' WITH DEFAULT_ROLE = 'role123' TENANTSETTING +CREATE USER 'u1'@'%' IDENTIFIED BY '123456' WITH DEFAULT_ROLE = 'role123', TENANTSETTING ---------- AST ------------ CreateUser( CreateUserStmt { @@ -8300,7 +8294,7 @@ Query( ---------- Input ---------- select * from @foo (pattern=>'[.]*parquet' file_format=>'tsv'); ---------- Output --------- -SELECT * FROM @foo ( FILE_FORMAT => 'tsv', PATTERN => '[.]*parquet', ) +SELECT * FROM '@foo' ( FILE_FORMAT => 'tsv', PATTERN => '[.]*parquet', ) ---------- AST ------------ Query( Query { @@ -8840,7 +8834,9 @@ CreateStage( create_option: Create, stage_name: "~", location: None, - file_format_options: {}, + file_format_options: FileFormatOptions { + options: {}, + }, on_error: "", size_limit: 0, validation_mode: "", @@ -8852,7 +8848,7 @@ CreateStage( ---------- Input ---------- CREATE STAGE IF NOT EXISTS test_stage 's3://load/files/' credentials=(aws_key_id='1a2b3c', aws_secret_key='4x5y6z') file_format=(type = CSV, compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE STAGE IF NOT EXISTS test_stage 's3://load/files/' CONNECTION = ( aws_key_id = '******b3c', aws_secret_key = '******y6z' ) FILE_FORMAT = (compression = 'GZIP', record_delimiter = ',', type = 'CSV' ) +CREATE STAGE IF NOT EXISTS test_stage 's3://load/files/' CONNECTION = ( aws_key_id = '1a2b3c', aws_secret_key = '4x5y6z' ) FILE_FORMAT = (compression = GZIP, record_delimiter = ',', type = CSV) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -8873,10 +8869,18 @@ CreateStage( }, }, ), - file_format_options: { - "compression": "GZIP", - "record_delimiter": ",", - "type": "CSV", + file_format_options: FileFormatOptions { + options: { + "compression": Keyword( + "GZIP", + ), + "record_delimiter": String( + ",", + ), + "type": Keyword( + "CSV", + ), + }, }, on_error: "", size_limit: 0, @@ -8889,7 +8893,7 @@ CreateStage( ---------- Input ---------- CREATE STAGE IF NOT EXISTS test_stage url='s3://load/files/' credentials=(aws_key_id='1a2b3c', aws_secret_key='4x5y6z') file_format=(type = CSV, compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE STAGE IF NOT EXISTS test_stage 's3://load/files/' CONNECTION = ( aws_key_id = '******b3c', aws_secret_key = '******y6z' ) FILE_FORMAT = (compression = 'GZIP', record_delimiter = ',', type = 'CSV' ) +CREATE STAGE IF NOT EXISTS test_stage 's3://load/files/' CONNECTION = ( aws_key_id = '1a2b3c', aws_secret_key = '4x5y6z' ) FILE_FORMAT = (compression = GZIP, record_delimiter = ',', type = CSV) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -8910,10 +8914,18 @@ CreateStage( }, }, ), - file_format_options: { - "compression": "GZIP", - "record_delimiter": ",", - "type": "CSV", + file_format_options: FileFormatOptions { + options: { + "compression": Keyword( + "GZIP", + ), + "record_delimiter": String( + ",", + ), + "type": Keyword( + "CSV", + ), + }, }, on_error: "", size_limit: 0, @@ -8926,7 +8938,7 @@ CreateStage( ---------- Input ---------- CREATE STAGE IF NOT EXISTS test_stage url='azblob://load/files/' connection=(account_name='1a2b3c' account_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE STAGE IF NOT EXISTS test_stage 'azblob://load/files/' CONNECTION = ( account_key = '******y6z', account_name = '******b3c' ) FILE_FORMAT = (compression = 'GZIP', record_delimiter = ',', type = 'CSV' ) +CREATE STAGE IF NOT EXISTS test_stage 'azblob://load/files/' CONNECTION = ( account_key = '4x5y6z', account_name = '1a2b3c' ) FILE_FORMAT = (compression = GZIP, record_delimiter = ',', type = CSV) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -8947,10 +8959,18 @@ CreateStage( }, }, ), - file_format_options: { - "compression": "GZIP", - "record_delimiter": ",", - "type": "CSV", + file_format_options: FileFormatOptions { + options: { + "compression": Keyword( + "GZIP", + ), + "record_delimiter": String( + ",", + ), + "type": Keyword( + "CSV", + ), + }, }, on_error: "", size_limit: 0, @@ -8963,7 +8983,7 @@ CreateStage( ---------- Input ---------- CREATE OR REPLACE STAGE test_stage url='azblob://load/files/' connection=(account_name='1a2b3c' account_key='4x5y6z') file_format=(type = CSV compression = GZIP record_delimiter=',') ---------- Output --------- -CREATE OR REPLACE STAGE test_stage 'azblob://load/files/' CONNECTION = ( account_key = '******y6z', account_name = '******b3c' ) FILE_FORMAT = (compression = 'GZIP', record_delimiter = ',', type = 'CSV' ) +CREATE OR REPLACE STAGE test_stage 'azblob://load/files/' CONNECTION = ( account_key = '4x5y6z', account_name = '1a2b3c' ) FILE_FORMAT = (compression = GZIP, record_delimiter = ',', type = CSV) ---------- AST ------------ CreateStage( CreateStageStmt { @@ -8984,10 +9004,18 @@ CreateStage( }, }, ), - file_format_options: { - "compression": "GZIP", - "record_delimiter": ",", - "type": "CSV", + file_format_options: FileFormatOptions { + options: { + "compression": Keyword( + "GZIP", + ), + "record_delimiter": String( + ",", + ), + "type": Keyword( + "CSV", + ), + }, }, on_error: "", size_limit: 0, @@ -9000,7 +9028,7 @@ CreateStage( ---------- Input ---------- DROP STAGE abc ---------- Output --------- -DROP STAGES abc +DROP STAGE abc ---------- AST ------------ DropStage { if_exists: false, @@ -9011,7 +9039,7 @@ DropStage { ---------- Input ---------- DROP STAGE ~ ---------- Output --------- -DROP STAGES ~ +DROP STAGE ~ ---------- AST ------------ DropStage { if_exists: false, @@ -9205,7 +9233,7 @@ OptimizeTable( ---------- Input ---------- OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5') LIMIT 10; ---------- Output --------- -OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => 9828b23f74664ff3806f44bbc1925ea5) LIMIT 10 +OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5') LIMIT 10 ---------- AST ------------ OptimizeTable( OptimizeTableStmt { @@ -9236,7 +9264,7 @@ OptimizeTable( ---------- Input ---------- OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10; ---------- Output --------- -OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10 +OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10 ---------- AST ------------ OptimizeTable( OptimizeTableStmt { @@ -9281,7 +9309,7 @@ OptimizeTable( ---------- Input ---------- ALTER TABLE t CLUSTER BY(c1); ---------- Output --------- -ALTER TABLE t CLUSTER BY c1 +ALTER TABLE t CLUSTER BY (c1) ---------- AST ------------ AlterTable( AlterTableStmt { @@ -9896,7 +9924,7 @@ AlterTable( ---------- Input ---------- ALTER TABLE t MODIFY COLUMN a int DEFAULT 1, COLUMN b float; ---------- Output --------- -ALTER TABLE t MODIFY COLUMN a Int32 DEFAULT 1, COLUMN b Float32 +ALTER TABLE t MODIFY COLUMN a Int32 DEFAULT 1, b Float32 ---------- AST ------------ AlterTable( AlterTableStmt { @@ -9971,7 +9999,7 @@ AlterTable( ---------- Input ---------- ALTER TABLE t MODIFY COLUMN a int NULL DEFAULT 1, COLUMN b float NOT NULL COMMENT 'column b'; ---------- Output --------- -ALTER TABLE t MODIFY COLUMN a Int32 NULL DEFAULT 1, COLUMN b Float32 NOT NULL COMMENT column b +ALTER TABLE t MODIFY COLUMN a Int32 NULL DEFAULT 1, b Float32 NOT NULL COMMENT 'column b' ---------- AST ------------ AlterTable( AlterTableStmt { @@ -10685,7 +10713,7 @@ Grant( ---------- Input ---------- GRANT SELECT, CREATE ON * TO ROLE role1; ---------- Output --------- -GRANT SELECT, CREATE ON * TO ROLE role1 +GRANT SELECT, CREATE ON * TO ROLE 'role1' ---------- AST ------------ Grant( GrantStmt { @@ -10728,7 +10756,7 @@ Grant( ---------- Input ---------- GRANT ALL ON *.* TO ROLE role2; ---------- Output --------- -GRANT ALL PRIVILEGES ON *.* TO ROLE role2 +GRANT ALL PRIVILEGES ON *.* TO ROLE 'role2' ---------- AST ------------ Grant( GrantStmt { @@ -10767,7 +10795,7 @@ Grant( ---------- Input ---------- GRANT ALL PRIVILEGES ON * TO ROLE role3; ---------- Output --------- -GRANT ALL PRIVILEGES ON * TO ROLE role3 +GRANT ALL PRIVILEGES ON * TO ROLE 'role3' ---------- AST ------------ Grant( GrantStmt { @@ -10826,7 +10854,7 @@ Grant( ---------- Input ---------- GRANT ROLE test TO ROLE `test-user`; ---------- Output --------- -GRANT ROLE test TO ROLE test-user +GRANT ROLE test TO ROLE 'test-user' ---------- AST ------------ Grant( GrantStmt { @@ -10897,7 +10925,7 @@ Grant( ---------- Input ---------- GRANT SELECT ON db01.* TO ROLE role1 ---------- Output --------- -GRANT SELECT ON db01.* TO ROLE role1 +GRANT SELECT ON db01.* TO ROLE 'role1' ---------- AST ------------ Grant( GrantStmt { @@ -10977,7 +11005,7 @@ Grant( ---------- Input ---------- GRANT SELECT ON db01.tb1 TO ROLE role1; ---------- Output --------- -GRANT SELECT ON db01.tb1 TO ROLE role1 +GRANT SELECT ON db01.tb1 TO ROLE 'role1' ---------- AST ------------ Grant( GrantStmt { @@ -11002,7 +11030,7 @@ Grant( ---------- Input ---------- GRANT SELECT ON tb1 TO ROLE role1; ---------- Output --------- -GRANT SELECT ON tb1 TO ROLE role1 +GRANT SELECT ON tb1 TO ROLE 'role1' ---------- AST ------------ Grant( GrantStmt { @@ -11092,7 +11120,7 @@ ShowGrants { ---------- Input ---------- SHOW GRANTS FOR ROLE role1; ---------- Output --------- -SHOW GRANTS FOR ROLE role1 +SHOW GRANTS FOR ROLE 'role1' ---------- AST ------------ ShowGrants { principal: Some( @@ -11106,7 +11134,7 @@ ShowGrants { ---------- Input ---------- SHOW GRANTS FOR ROLE 'role1'; ---------- Output --------- -SHOW GRANTS FOR ROLE role1 +SHOW GRANTS FOR ROLE 'role1' ---------- AST ------------ ShowGrants { principal: Some( @@ -11146,7 +11174,7 @@ Revoke( ---------- Input ---------- REVOKE SELECT ON tb1 FROM ROLE role1; ---------- Output --------- -REVOKE SELECT ON tb1 FROM ROLE role1 +REVOKE SELECT ON tb1 FROM ROLE 'role1' ---------- AST ------------ Revoke( RevokeStmt { @@ -11169,7 +11197,7 @@ Revoke( ---------- Input ---------- REVOKE SELECT ON tb1 FROM ROLE 'role1'; ---------- Output --------- -REVOKE SELECT ON tb1 FROM ROLE role1 +REVOKE SELECT ON tb1 FROM ROLE 'role1' ---------- AST ------------ Revoke( RevokeStmt { @@ -11203,7 +11231,7 @@ DropRole { ---------- Input ---------- GRANT ROLE test TO ROLE 'test-user'; ---------- Output --------- -GRANT ROLE test TO ROLE test-user +GRANT ROLE test TO ROLE 'test-user' ---------- AST ------------ Grant( GrantStmt { @@ -11220,7 +11248,7 @@ Grant( ---------- Input ---------- GRANT ROLE test TO ROLE `test-user`; ---------- Output --------- -GRANT ROLE test TO ROLE test-user +GRANT ROLE test TO ROLE 'test-user' ---------- AST ------------ Grant( GrantStmt { @@ -11237,7 +11265,7 @@ Grant( ---------- Input ---------- SET ROLE `test-user`; ---------- Output --------- -SET ROLE test-user +SET ROLE 'test-user' ---------- AST ------------ SetRole { is_default: false, @@ -11248,7 +11276,7 @@ SetRole { ---------- Input ---------- SET ROLE 'test-user'; ---------- Output --------- -SET ROLE test-user +SET ROLE 'test-user' ---------- AST ------------ SetRole { is_default: false, @@ -11259,7 +11287,7 @@ SetRole { ---------- Input ---------- SET ROLE ROLE1; ---------- Output --------- -SET ROLE ROLE1 +SET ROLE 'ROLE1' ---------- AST ------------ SetRole { is_default: false, @@ -11295,7 +11323,7 @@ COPY INTO mytable FROM '@~/mybucket/my data.csv' size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @~/mybucket/my data.csv SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@~/mybucket/my data.csv' SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11318,7 +11346,9 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: {}, + file_format: FileFormatOptions { + options: {}, + }, files: None, pattern: None, force: false, @@ -11345,8 +11375,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @~/mybucket/data.csv FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@~/mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11369,11 +11399,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11403,7 +11443,7 @@ COPY INTO mytable max_files=10; ---------- Output --------- COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 MAX_FILES = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11435,11 +11475,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11469,7 +11519,7 @@ COPY INTO mytable max_files=3000; ---------- Output --------- COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 MAX_FILES = 3000 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 3000 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11501,11 +11551,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11536,8 +11596,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = '******900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11571,11 +11631,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11606,8 +11676,8 @@ COPY INTO mytable skip_header = 1 ); ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = '******900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11641,11 +11711,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11698,7 +11778,9 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: {}, + file_format: FileFormatOptions { + options: {}, + }, files: None, pattern: None, force: false, @@ -11750,7 +11832,9 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: {}, + file_format: FileFormatOptions { + options: {}, + }, files: None, pattern: None, force: false, @@ -11778,8 +11862,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @my_stage FILE_FORMAT = (error_on_column_count_mismatch = 'false', field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@my_stage' FILE_FORMAT = (error_on_column_count_mismatch = false, field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11802,12 +11886,24 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "error_on_column_count_mismatch": "false", - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "error_on_column_count_mismatch": Bool( + false, + ), + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -11835,7 +11931,7 @@ COPY INTO 's3://mybucket/data.csv' ) ---------- Output --------- COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CopyIntoLocation( CopyIntoLocationStmt { @@ -11866,11 +11962,21 @@ CopyIntoLocation( }, }, ), - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, single: false, max_file_size: 0, @@ -11883,7 +11989,7 @@ CopyIntoLocation( COPY INTO '@my_stage/my data' FROM mytable; ---------- Output --------- -COPY INTO @my_stage/my data FROM mytable SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +COPY INTO '@my_stage/my data' FROM mytable SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CopyIntoLocation( CopyIntoLocationStmt { @@ -11905,7 +12011,9 @@ CopyIntoLocation( dst: Stage( "my_stage/my data", ), - file_format: {}, + file_format: FileFormatOptions { + options: {}, + }, single: false, max_file_size: 0, detailed_output: false, @@ -11923,8 +12031,8 @@ COPY INTO @my_stage skip_header = 1 ); ---------- Output --------- -COPY INTO @my_stage FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +COPY INTO '@my_stage' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CopyIntoLocation( CopyIntoLocationStmt { @@ -11946,11 +12054,21 @@ CopyIntoLocation( dst: Stage( "my_stage", ), - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, single: false, max_file_size: 0, @@ -11974,8 +12092,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id = '******key', aws_secret_key = '******key' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id = 'access_key', aws_secret_key = 'secret_key' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12010,11 +12128,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12042,8 +12170,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12066,11 +12194,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12098,8 +12236,8 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/dir/ FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/dir/' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12122,11 +12260,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12154,8 +12302,8 @@ COPY INTO mytable ) force=true; ---------- Output --------- -COPY INTO mytable FROM @external_stage/path/to/file.csv FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV) PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12178,11 +12326,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12212,7 +12370,7 @@ COPY INTO mytable disable_variant_check=true; ---------- Output --------- COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = '1', type = 'CSV') SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = abort +', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12244,11 +12402,21 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + file_format: FileFormatOptions { + options: { + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12275,7 +12443,7 @@ COPY INTO books FROM 's3://databend/books.csv' ) FILE_FORMAT = (type = CSV); ---------- Output --------- -COPY INTO books FROM 's3://databend/books.csv' CONNECTION = ( access_key_id = '******SER', endpoint_url = '******00/', region = '******t-2', secret_access_key = '******123' ) FILE_FORMAT = (type = 'CSV') PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO books FROM 's3://databend/books.csv' CONNECTION = ( access_key_id = 'ROOTUSER', endpoint_url = 'http://localhost:9000/', region = 'us-west-2', secret_access_key = 'CHANGEME123' ) FILE_FORMAT = (type = CSV) PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12312,8 +12480,12 @@ CopyIntoTable( }, dst_columns: None, hints: None, - file_format: { - "type": "CSV", + file_format: FileFormatOptions { + options: { + "type": Keyword( + "CSV", + ), + }, }, files: None, pattern: None, @@ -12807,7 +12979,7 @@ ShowIndexes { ---------- Input ---------- PRESIGN @my_stage ---------- Output --------- -PRESIGN DOWNLOAD my_stage EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12824,7 +12996,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/path/to/dir/ ---------- Output --------- -PRESIGN DOWNLOAD my_stage/path/to/dir/ EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/path/to/dir/ EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12841,7 +13013,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/path/to/file ---------- Output --------- -PRESIGN DOWNLOAD my_stage/path/to/file EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/path/to/file EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12858,7 +13030,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/my\ file.csv ---------- Output --------- -PRESIGN DOWNLOAD my_stage/my file.csv EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/my\ file.csv EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12875,7 +13047,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/\"file\".csv ---------- Output --------- -PRESIGN DOWNLOAD my_stage/"file".csv EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/\"file\".csv EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12892,7 +13064,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/\'file\'.csv ---------- Output --------- -PRESIGN DOWNLOAD my_stage/'file'.csv EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/\'file\'.csv EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12909,7 +13081,7 @@ Presign( ---------- Input ---------- PRESIGN @my_stage/\\file\\.csv ---------- Output --------- -PRESIGN DOWNLOAD my_stage/\file\.csv EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/\\file\\.csv EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12926,7 +13098,7 @@ Presign( ---------- Input ---------- PRESIGN DOWNLOAD @my_stage/path/to/file ---------- Output --------- -PRESIGN DOWNLOAD my_stage/path/to/file EXPIRE = 3600 +PRESIGN DOWNLOAD @my_stage/path/to/file EXPIRE = 3600 ---------- AST ------------ Presign( PresignStmt { @@ -12943,7 +13115,7 @@ Presign( ---------- Input ---------- PRESIGN UPLOAD @my_stage/path/to/file EXPIRE=7200 ---------- Output --------- -PRESIGN UPLOAD my_stage/path/to/file EXPIRE = 7200 +PRESIGN UPLOAD @my_stage/path/to/file EXPIRE = 7200 ---------- AST ------------ Presign( PresignStmt { @@ -12960,7 +13132,7 @@ Presign( ---------- Input ---------- PRESIGN UPLOAD @my_stage/path/to/file EXPIRE=7200 CONTENT_TYPE='application/octet-stream' ---------- Output --------- -PRESIGN UPLOAD my_stage/path/to/file EXPIRE = 7200 CONTENT_TYPE = 'application/octet-stream' +PRESIGN UPLOAD @my_stage/path/to/file EXPIRE = 7200 CONTENT_TYPE = 'application/octet-stream' ---------- AST ------------ Presign( PresignStmt { @@ -12979,7 +13151,7 @@ Presign( ---------- Input ---------- PRESIGN UPLOAD @my_stage/path/to/file CONTENT_TYPE='application/octet-stream' EXPIRE=7200 ---------- Output --------- -PRESIGN UPLOAD my_stage/path/to/file EXPIRE = 7200 CONTENT_TYPE = 'application/octet-stream' +PRESIGN UPLOAD @my_stage/path/to/file EXPIRE = 7200 CONTENT_TYPE = 'application/octet-stream' ---------- AST ------------ Presign( PresignStmt { @@ -12998,7 +13170,7 @@ Presign( ---------- Input ---------- CREATE SHARE ENDPOINT IF NOT EXISTS t URL='http://127.0.0.1' TENANT=x ARGS=(jwks_key_file="https://eks.public/keys" ssl_cert="cert.pem") COMMENT='share endpoint comment'; ---------- Output --------- -CREATE SHARE ENDPOINT IF NOT EXISTS t URL='http://127.0.0.1/' TENANT=x ARGS=(jwks_key_file=https://eks.public/keys,ssl_cert=cert.pem,) COMMENT = 'share endpoint comment' +CREATE SHARE ENDPOINT IF NOT EXISTS t URL='http://127.0.0.1/' TENANT=x ARGS=(jwks_key_file = 'https://eks.public/keys', ssl_cert = 'cert.pem') COMMENT = 'share endpoint comment' ---------- AST ------------ CreateShareEndpoint( CreateShareEndpointStmt { @@ -13043,7 +13215,7 @@ CreateShareEndpoint( ---------- Input ---------- CREATE OR REPLACE SHARE ENDPOINT t URL='http://127.0.0.1' TENANT=x ARGS=(jwks_key_file="https://eks.public/keys" ssl_cert="cert.pem") COMMENT='share endpoint comment'; ---------- Output --------- -CREATE OR REPLACE SHARE ENDPOINT t URL='http://127.0.0.1/' TENANT=x ARGS=(jwks_key_file=https://eks.public/keys,ssl_cert=cert.pem,) COMMENT = 'share endpoint comment' +CREATE OR REPLACE SHARE ENDPOINT t URL='http://127.0.0.1/' TENANT=x ARGS=(jwks_key_file = 'https://eks.public/keys', ssl_cert = 'cert.pem') COMMENT = 'share endpoint comment' ---------- AST ------------ CreateShareEndpoint( CreateShareEndpointStmt { @@ -13926,7 +14098,7 @@ UnSetVariable( ---------- Input ---------- select $1 FROM '@my_stage/my data/' ---------- Output --------- -SELECT $1 FROM @my_stage/my data/ +SELECT $1 FROM '@my_stage/my data/' ---------- AST ------------ Query( Query { @@ -14000,7 +14172,7 @@ Query( SELECT t.c1 FROM @stage1/dir/file ( file_format => 'PARQUET', FILES => ('file1', 'file2')) t; ---------- Output --------- -SELECT t.c1 FROM @stage1/dir/file ( FILES => ('file1', 'file2'), FILE_FORMAT => 'PARQUET', ) AS t +SELECT t.c1 FROM '@stage1/dir/file' ( FILES => ('file1', 'file2'), FILE_FORMAT => 'PARQUET', ) AS t ---------- AST ------------ Query( Query { @@ -14104,7 +14276,7 @@ select table0.c1, table1.c2 from @stage1/dir/file ( FILE_FORMAT => 'parquet', FILES => ('file1', 'file2')) table0 left join table1; ---------- Output --------- -SELECT table0.c1, table1.c2 FROM @stage1/dir/file ( FILES => ('file1', 'file2'), FILE_FORMAT => 'parquet', ) AS table0 LEFT OUTER JOIN table1 +SELECT table0.c1, table1.c2 FROM '@stage1/dir/file' ( FILES => ('file1', 'file2'), FILE_FORMAT => 'parquet', ) AS table0 LEFT OUTER JOIN table1 ---------- AST ------------ Query( Query { @@ -14366,17 +14538,26 @@ Query( CREATE FILE FORMAT my_csv type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1; ---------- Output --------- -CREATE FILE_FORMAT my_csv {"field_delimiter": ",", "record_delimiter": "\n", "skip_header": "1", "type": "CSV"} +CREATE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV ---------- AST ------------ CreateFileFormat { create_option: Create, name: "my_csv", - file_format_options: FileFormatOptionsAst { + file_format_options: FileFormatOptions { options: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), }, }, } @@ -14386,17 +14567,26 @@ CreateFileFormat { CREATE OR REPLACE FILE FORMAT my_csv type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1; ---------- Output --------- -CREATE OR REPLACE FILE_FORMAT my_csv {"field_delimiter": ",", "record_delimiter": "\n", "skip_header": "1", "type": "CSV"} +CREATE OR REPLACE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = ' +', skip_header = 1, type = CSV ---------- AST ------------ CreateFileFormat { create_option: CreateOrReplace, name: "my_csv", - file_format_options: FileFormatOptionsAst { + file_format_options: FileFormatOptions { options: { - "field_delimiter": ",", - "record_delimiter": "\n", - "skip_header": "1", - "type": "CSV", + "field_delimiter": String( + ",", + ), + "record_delimiter": String( + "\n", + ), + "skip_header": U64( + 1, + ), + "type": Keyword( + "CSV", + ), }, }, } @@ -14413,7 +14603,7 @@ ShowFileFormats ---------- Input ---------- DROP FILE FORMAT my_csv ---------- Output --------- -DROP FILE_FORMAT my_csv +DROP FILE FORMAT my_csv ---------- AST ------------ DropFileFormat { if_exists: false, @@ -16027,7 +16217,7 @@ AlterNetworkPolicy( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 15 MINUTE SUSPEND_TASK_AFTER_NUM_FAILURES = 3 ERROR_INTEGRATION = 'notification_name' COMMENT = 'This is test task 1' DATABASE = 'target', TIMEZONE = 'America/Los Angeles' AS SELECT * FROM MyTable1 ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = MyWarehouse SCHEDULE 900 SECOND database = 'target' timezone = 'America/Los Angeles' SUSPEND_TASK_AFTER 3 FAILURES COMMENTS = 'This is test task 1' ERROR INTEGRATION = 'notification_name' AS SELECT * FROM MyTable1 +CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 900 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 3 ERROR_INTEGRATION = 'notification_name' COMMENTS = 'This is test task 1'database = 'target', timezone = 'America/Los Angeles' AS SELECT * FROM MyTable1 ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16053,7 +16243,9 @@ CreateTask( error_integration: Some( "notification_name", ), - comments: "This is test task 1", + comments: Some( + "This is test task 1", + ), after: [], when_condition: None, sql: SingleStatement( @@ -16066,7 +16258,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 15 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 3 COMMENT = 'This is test task 1' AS SELECT * FROM MyTable1 ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = MyWarehouse SCHEDULE 15 SECOND SUSPEND_TASK_AFTER 3 FAILURES COMMENTS = 'This is test task 1' AS SELECT * FROM MyTable1 +CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 15 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 3 COMMENTS = 'This is test task 1' AS SELECT * FROM MyTable1 ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16087,7 +16279,9 @@ CreateTask( 3, ), error_integration: None, - comments: "This is test task 1", + comments: Some( + "This is test task 1", + ), after: [], when_condition: None, sql: SingleStatement( @@ -16100,7 +16294,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 1215 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 3 COMMENT = 'This is test task 1' AS SELECT * FROM MyTable1 ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = MyWarehouse SCHEDULE 1215 SECOND SUSPEND_TASK_AFTER 3 FAILURES COMMENTS = 'This is test task 1' AS SELECT * FROM MyTable1 +CREATE TASK IF NOT EXISTS MyTask1 WAREHOUSE = 'MyWarehouse' SCHEDULE = 1215 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 3 COMMENTS = 'This is test task 1' AS SELECT * FROM MyTable1 ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16121,7 +16315,9 @@ CreateTask( 3, ), error_integration: None, - comments: "This is test task 1", + comments: Some( + "This is test task 1", + ), after: [], when_condition: None, sql: SingleStatement( @@ -16134,7 +16330,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 6 * * *' 'America/Los_Angeles' COMMENT = 'serverless + cron' AS insert into t (c1, c2) values (1, 2), (3, 4) ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE CRON '0 6 * * *' TIMEZONE 'America/Los_Angeles' COMMENTS = 'serverless + cron' AS INSERT INTO t (c1, c2) VALUES (1, 2), (3, 4) +CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 6 * * *' 'America/Los_Angeles' COMMENTS = 'serverless + cron' AS INSERT INTO t (c1, c2) VALUES (1, 2), (3, 4) ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16154,7 +16350,9 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "serverless + cron", + comments: Some( + "serverless + cron", + ), after: [], when_condition: None, sql: SingleStatement( @@ -16167,7 +16365,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 12 * * *' AS copy into streams_test.paper_table from @stream_stage FILE_FORMAT = (TYPE = PARQUET) PURGE=true ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE CRON '0 12 * * *' AS COPY INTO streams_test.paper_table FROM @stream_stage FILE_FORMAT = (type = 'PARQUET') PURGE = true FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 12 * * *' AS COPY INTO streams_test.paper_table FROM '@stream_stage' FILE_FORMAT = (type = PARQUET) PURGE = true FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16185,11 +16383,11 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: SingleStatement( - "COPY INTO streams_test.paper_table FROM @stream_stage FILE_FORMAT = (type = 'PARQUET') PURGE = true FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort", + "COPY INTO streams_test.paper_table FROM '@stream_stage' FILE_FORMAT = (type = PARQUET) PURGE = true FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort", ), }, ) @@ -16198,7 +16396,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 13 * * *' AS COPY INTO @my_internal_stage FROM canadian_city_population FILE_FORMAT = (TYPE = PARQUET) ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE CRON '0 13 * * *' AS COPY INTO @my_internal_stage FROM canadian_city_population FILE_FORMAT = (type = 'PARQUET') SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +CREATE TASK IF NOT EXISTS MyTask1 SCHEDULE = USING CRON '0 13 * * *' AS COPY INTO '@my_internal_stage' FROM canadian_city_population FILE_FORMAT = (type = PARQUET) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16216,11 +16414,11 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: SingleStatement( - "COPY INTO @my_internal_stage FROM canadian_city_population FILE_FORMAT = (type = 'PARQUET') SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false", + "COPY INTO '@my_internal_stage' FROM canadian_city_population FILE_FORMAT = (type = PARQUET) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false", ), }, ) @@ -16229,7 +16427,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 AFTER 'task2', 'task3' WHEN SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') != 'VALIDATION' AS VACUUM TABLE t ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 AFTER = '["task2", "task3"]' WHEN = 'Some("(SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION')")' AS VACUUM TABLE t +CREATE TASK IF NOT EXISTS MyTask1 AFTER 'task2', 'task3' WHEN (SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION') AS VACUUM TABLE t ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16242,13 +16440,55 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [ "task2", "task3", ], when_condition: Some( - "(SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION')", + BinaryOp { + span: Some( + 111..113, + ), + op: NotEq, + left: FunctionCall { + span: Some( + 62..110, + ), + func: FunctionCall { + distinct: false, + name: Identifier { + span: Some( + 62..97, + ), + name: "SYSTEM$GET_PREDECESSOR_RETURN_VALUE", + quote: None, + is_hole: false, + }, + args: [ + Literal { + span: Some( + 98..109, + ), + lit: String( + "task_name", + ), + }, + ], + params: [], + window: None, + lambda: None, + }, + }, + right: Literal { + span: Some( + 114..126, + ), + lit: String( + "VALIDATION", + ), + }, + }, ), sql: SingleStatement( "VACUUM TABLE t ", @@ -16260,7 +16500,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 DATABASE = 'target', TIMEZONE = 'America/Los Angeles' AS VACUUM TABLE t ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 database = 'target' timezone = 'America/Los Angeles' AS VACUUM TABLE t +CREATE TASK IF NOT EXISTS MyTask1 database = 'target', timezone = 'America/Los Angeles' AS VACUUM TABLE t ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -16276,7 +16516,7 @@ CreateTask( }, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: SingleStatement( @@ -16297,7 +16537,7 @@ CREATE TASK IF NOT EXISTS MyTask1 DATABASE = 'target', TIMEZONE = 'America/Los A commit; END ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 database = 'target' timezone = 'America/Los Angeles' AS BEGIN +CREATE TASK IF NOT EXISTS MyTask1 database = 'target', timezone = 'America/Los Angeles' AS BEGIN BEGIN; INSERT INTO t VALUES ('a;'); DELETE FROM t WHERE (c = ';'); @@ -16320,7 +16560,7 @@ CreateTask( }, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: ScriptBlock( @@ -16349,7 +16589,7 @@ CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = 'test-parser' SCHEDULE = 1 SECO *; END ---------- Output --------- -CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = test-parser SCHEDULE 1 SECOND AS BEGIN +CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = 'test-parser' SCHEDULE = 1 SECOND AS BEGIN MERGE INTO t USING s ON (t.c = s.c) WHEN MATCHED THEN UPDATE * WHEN NOT MATCHED THEN INSERT *; END; ---------- AST ------------ @@ -16370,7 +16610,7 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: ScriptBlock( @@ -16393,7 +16633,7 @@ CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = 'test-parser' SCHEDULE = 1 SECO INSERT values('a;', 1, "str"); END ---------- Output --------- -CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = test-parser SCHEDULE 1 SECOND AS BEGIN +CREATE TASK IF NOT EXISTS merge_task WAREHOUSE = 'test-parser' SCHEDULE = 1 SECOND AS BEGIN MERGE INTO t USING s ON (t.c = s.c) WHEN MATCHED THEN UPDATE * WHEN NOT MATCHED THEN INSERT VALUES('a;', 1, "str"); END; ---------- AST ------------ @@ -16414,7 +16654,7 @@ CreateTask( session_parameters: {}, suspend_task_after_num_failures: None, error_integration: None, - comments: "", + comments: None, after: [], when_condition: None, sql: ScriptBlock( @@ -16457,7 +16697,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 ADD AFTER 'task2', 'task3' ---------- Output --------- -ALTER TASK MyTask1 ADD AFTER = '["task2", "task3"]' +ALTER TASK MyTask1 ADD AFTER 'task2', 'task3' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16476,7 +16716,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 REMOVE AFTER 'task2' ---------- Output --------- -ALTER TASK MyTask1 REMOVE AFTER = '["task2"]' +ALTER TASK MyTask1 REMOVE AFTER 'task2' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16494,7 +16734,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 SET WAREHOUSE= 'MyWarehouse' SCHEDULE = USING CRON '0 6 * * *' 'America/Los_Angeles' COMMENT = 'serverless + cron' ---------- Output --------- -ALTER TASK MyTask1 SET WAREHOUSE = MyWarehouse SET SCHEDULE CRON '0 6 * * *' TIMEZONE 'America/Los_Angeles' COMMENTS = 'serverless + cron' +ALTER TASK MyTask1 SET WAREHOUSE = 'MyWarehouse' SCHEDULE = USING CRON '0 6 * * *' 'America/Los_Angeles' COMMENT = 'serverless + cron' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16526,7 +16766,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 SET WAREHOUSE= 'MyWarehouse' SCHEDULE = 13 MINUTE SUSPEND_TASK_AFTER_NUM_FAILURES = 10 COMMENT = 'serverless + cron' ---------- Output --------- -ALTER TASK MyTask1 SET WAREHOUSE = MyWarehouse SET SCHEDULE 780 SECOND SUSPEND TASK AFTER 10 FAILURES COMMENTS = 'serverless + cron' +ALTER TASK MyTask1 SET WAREHOUSE = 'MyWarehouse' SCHEDULE = 780 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 10 COMMENT = 'serverless + cron' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16557,7 +16797,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 SET WAREHOUSE= 'MyWarehouse' SCHEDULE = 5 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 10 COMMENT = 'serverless + cron' ---------- Output --------- -ALTER TASK MyTask1 SET WAREHOUSE = MyWarehouse SET SCHEDULE 5 SECOND SUSPEND TASK AFTER 10 FAILURES COMMENTS = 'serverless + cron' +ALTER TASK MyTask1 SET WAREHOUSE = 'MyWarehouse' SCHEDULE = 5 SECOND SUSPEND_TASK_AFTER_NUM_FAILURES = 10 COMMENT = 'serverless + cron' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16588,7 +16828,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 SET DATABASE='newDB', TIMEZONE='America/Los_Angeles' ---------- Output --------- -ALTER TASK MyTask1 database = 'newDB' timezone = 'America/Los_Angeles' +ALTER TASK MyTask1 SET database = 'newDB', timezone = 'America/Los_Angeles' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16614,7 +16854,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 SET ERROR_INTEGRATION = 'candidate_notifictaion' ---------- Output --------- -ALTER TASK MyTask1 ERROR INTEGRATION = 'candidate_notifictaion' +ALTER TASK MyTask1 SET ERROR_INTEGRATION = 'candidate_notifictaion' ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16637,7 +16877,7 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask2 MODIFY AS SELECT CURRENT_VERSION() ---------- Output --------- -ALTER TASK MyTask2 AS SELECT CURRENT_VERSION() +ALTER TASK MyTask2 MODIFY AS SELECT CURRENT_VERSION() ---------- AST ------------ AlterTask( AlterTaskStmt { @@ -16663,7 +16903,7 @@ ALTER TASK MyTask2 MODIFY AS commit; END ---------- Output --------- -ALTER TASK MyTask2 AS BEGIN +ALTER TASK MyTask2 MODIFY AS BEGIN BEGIN; INSERT INTO t VALUES ('a;'); DELETE FROM t WHERE (c = ';'); @@ -16695,14 +16935,56 @@ AlterTask( ---------- Input ---------- ALTER TASK MyTask1 MODIFY WHEN SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') != 'VALIDATION' ---------- Output --------- -ALTER TASK MyTask1 WHEN (SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION') +ALTER TASK MyTask1 MODIFY WHEN (SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION') ---------- AST ------------ AlterTask( AlterTaskStmt { if_exists: false, name: "MyTask1", options: ModifyWhen( - "(SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION')", + BinaryOp { + span: Some( + 80..82, + ), + op: NotEq, + left: FunctionCall { + span: Some( + 31..79, + ), + func: FunctionCall { + distinct: false, + name: Identifier { + span: Some( + 31..66, + ), + name: "SYSTEM$GET_PREDECESSOR_RETURN_VALUE", + quote: None, + is_hole: false, + }, + args: [ + Literal { + span: Some( + 67..78, + ), + lit: String( + "task_name", + ), + }, + ], + params: [], + window: None, + lambda: None, + }, + }, + right: Literal { + span: Some( + 83..95, + ), + lit: String( + "VALIDATION", + ), + }, + }, ), }, ) @@ -16760,7 +17042,7 @@ DescribeTask( ---------- Input ---------- CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE='s3' ---------- Output --------- -CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE = s3 +CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE = 's3' ---------- AST ------------ CreateConnection( CreateConnectionStmt { @@ -16782,7 +17064,7 @@ CreateConnection( ---------- Input ---------- CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE='s3' any_arg='any_value' ---------- Output --------- -CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE = s3 any_arg = ******lue +CREATE CONNECTION IF NOT EXISTS my_conn STORAGE_TYPE = 's3' any_arg = 'any_value' ---------- AST ------------ CreateConnection( CreateConnectionStmt { @@ -16806,7 +17088,7 @@ CreateConnection( ---------- Input ---------- CREATE OR REPLACE CONNECTION my_conn STORAGE_TYPE='s3' any_arg='any_value' ---------- Output --------- -CREATE OR REPLACE CONNECTION my_conn STORAGE_TYPE = s3 any_arg = ******lue +CREATE OR REPLACE CONNECTION my_conn STORAGE_TYPE = 's3' any_arg = 'any_value' ---------- AST ------------ CreateConnection( CreateConnectionStmt { @@ -16830,7 +17112,7 @@ CreateConnection( ---------- Input ---------- DROP CONNECTION IF EXISTS my_conn; ---------- Output --------- -CREATE CONNECTION IF NOT EXISTS my_conn +DROP CONNECTION IF EXISTS my_conn ---------- AST ------------ DropConnection( DropConnectionStmt { @@ -16850,7 +17132,7 @@ DropConnection( ---------- Input ---------- DESC CONNECTION my_conn; ---------- Output --------- -CREATE CONNECTION my_conn +DESCRIBE CONNECTION my_conn ---------- AST ------------ DescribeConnection( DescribeConnectionStmt { @@ -16892,7 +17174,7 @@ ShowLocks( ---------- Input ---------- CREATE PIPE IF NOT EXISTS MyPipe1 AUTO_INGEST = TRUE COMMENT = 'This is test pipe 1' AS COPY INTO MyTable1 FROM '@~/MyStage1' FILE_FORMAT = (TYPE = 'CSV') ---------- Output --------- -CREATE PIPE IF NOT EXISTS MyPipe1 AUTO_INGEST = TRUE COMMENTS = 'This is test pipe 1' AS COPY INTO MyTable1 FROM @~/MyStage1 FILE_FORMAT = (type = 'CSV') PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +CREATE PIPE IF NOT EXISTS MyPipe1 AUTO_INGEST = TRUE COMMENTS = 'This is test pipe 1' AS COPY INTO MyTable1 FROM '@~/MyStage1' FILE_FORMAT = (type = 'CSV') PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CreatePipe( CreatePipeStmt { @@ -16920,8 +17202,12 @@ CreatePipe( }, dst_columns: None, hints: None, - file_format: { - "type": "CSV", + file_format: FileFormatOptions { + options: { + "type": String( + "CSV", + ), + }, }, files: None, pattern: None, @@ -16942,7 +17228,7 @@ CreatePipe( ---------- Input ---------- CREATE PIPE pipe1 AS COPY INTO db1.MyTable1 FROM @~/mybucket/data.csv ---------- Output --------- -CREATE PIPE pipe1 AS COPY INTO db1.MyTable1 FROM @~/mybucket/data.csv PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +CREATE PIPE pipe1 AS COPY INTO db1.MyTable1 FROM '@~/mybucket/data.csv' PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CreatePipe( CreatePipeStmt { @@ -16979,7 +17265,9 @@ CreatePipe( }, dst_columns: None, hints: None, - file_format: {}, + file_format: FileFormatOptions { + options: {}, + }, files: None, pattern: None, force: false, @@ -17100,7 +17388,7 @@ DescribePipe( ---------- Input ---------- CREATE NOTIFICATION INTEGRATION IF NOT EXISTS SampleNotification type = webhook enabled = true webhook = (url = 'https://example.com', method = 'GET', authorization_header = 'bearer auth') ---------- Output --------- -CREATE NOTIFICATION INTEGRATION IF NOT EXISTS SampleNotification TYPE = webhook WEBHOOK = (URL = 'https://example.com' METHOD = 'GET' AUTHORIZATION_HEADER = 'bearer auth' ) +CREATE NOTIFICATION INTEGRATION IF NOT EXISTS SampleNotification TYPE = webhook ENABLED = true WEBHOOK = ( URL = 'https://example.com' METHOD = 'GET' AUTHORIZATION_HEADER = 'bearer auth' ) ---------- AST ------------ CreateNotification( CreateNotificationStmt { @@ -17121,7 +17409,7 @@ CreateNotification( ), }, ), - comments: "", + comments: None, }, ) @@ -17129,7 +17417,7 @@ CreateNotification( ---------- Input ---------- CREATE NOTIFICATION INTEGRATION SampleNotification type = webhook enabled = true webhook = (url = 'https://example.com') COMMENT = 'notify' ---------- Output --------- -CREATE NOTIFICATION INTEGRATION SampleNotification TYPE = webhook WEBHOOK = (URL = 'https://example.com' ) COMMENTS = 'notify' +CREATE NOTIFICATION INTEGRATION SampleNotification TYPE = webhook ENABLED = true WEBHOOK = ( URL = 'https://example.com' ) COMMENTS = 'notify' ---------- AST ------------ CreateNotification( CreateNotificationStmt { @@ -17146,7 +17434,9 @@ CreateNotification( authorization_header: None, }, ), - comments: "notify", + comments: Some( + "notify", + ), }, ) @@ -17176,7 +17466,7 @@ AlterNotification( ---------- Input ---------- ALTER NOTIFICATION INTEGRATION SampleNotification SET webhook = (url = 'https://example.com') ---------- Output --------- -ALTER NOTIFICATION INTEGRATION SampleNotification SET WEBHOOK = (URL = 'https://example.com' ) +ALTER NOTIFICATION INTEGRATION SampleNotification SET WEBHOOK = ( URL = 'https://example.com' ) ---------- AST ------------ AlterNotification( AlterNotificationStmt { @@ -17395,7 +17685,7 @@ Query( ---------- Input ---------- GRANT OWNERSHIP ON d20_0014.* TO ROLE 'd20_0015_owner'; ---------- Output --------- -GRANT OWNERSHIP ON d20_0014.* TO ROLE d20_0015_owner +GRANT OWNERSHIP ON d20_0014.* TO ROLE 'd20_0015_owner' ---------- AST ------------ Grant( GrantStmt { @@ -17419,7 +17709,7 @@ Grant( ---------- Input ---------- GRANT OWNERSHIP ON d20_0014.t TO ROLE 'd20_0015_owner'; ---------- Output --------- -GRANT OWNERSHIP ON d20_0014.t TO ROLE d20_0015_owner +GRANT OWNERSHIP ON d20_0014.t TO ROLE 'd20_0015_owner' ---------- AST ------------ Grant( GrantStmt { @@ -17444,7 +17734,7 @@ Grant( ---------- Input ---------- GRANT OWNERSHIP ON STAGE s1 TO ROLE 'd20_0015_owner'; ---------- Output --------- -GRANT OWNERSHIP ON STAGE s1 TO ROLE d20_0015_owner +GRANT OWNERSHIP ON STAGE s1 TO ROLE 'd20_0015_owner' ---------- AST ------------ Grant( GrantStmt { @@ -17466,7 +17756,7 @@ Grant( ---------- Input ---------- GRANT OWNERSHIP ON UDF f1 TO ROLE 'd20_0015_owner'; ---------- Output --------- -GRANT OWNERSHIP ON UDF f1 TO ROLE d20_0015_owner +GRANT OWNERSHIP ON UDF f1 TO ROLE 'd20_0015_owner' ---------- AST ------------ Grant( GrantStmt { @@ -17648,7 +17938,7 @@ CreateUDF( ---------- Input ---------- CREATE FUNCTION binary_reverse (BINARY) RETURNS BINARY LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815'; ---------- Output --------- -CREATE FUNCTION binary_reverse (BINARY NULL) RETURNS BINARY NULL LANGUAGE python HANDLER = binary_reverse ADDRESS = http://0.0.0.0:8815 +CREATE FUNCTION binary_reverse (BINARY NULL) RETURNS BINARY NULL LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815' ---------- AST ------------ CreateUDF( CreateUDFStmt { @@ -17682,7 +17972,7 @@ CreateUDF( ---------- Input ---------- CREATE OR REPLACE FUNCTION binary_reverse (BINARY) RETURNS BINARY LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815'; ---------- Output --------- -CREATE OR REPLACE FUNCTION binary_reverse (BINARY NULL) RETURNS BINARY NULL LANGUAGE python HANDLER = binary_reverse ADDRESS = http://0.0.0.0:8815 +CREATE OR REPLACE FUNCTION binary_reverse (BINARY NULL) RETURNS BINARY NULL LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815' ---------- AST ------------ CreateUDF( CreateUDFStmt { @@ -17724,7 +18014,7 @@ def addone_py(i): return i+1 $$; ---------- Output --------- -CREATE OR REPLACE FUNCTION addone (Int32 NULL) RETURNS Int32 NULL LANGUAGE python runtime_version = HANDLER = addone_py AS $$ +CREATE OR REPLACE FUNCTION addone (Int32 NULL) RETURNS Int32 NULL LANGUAGE python HANDLER = 'addone_py' AS $$ def addone_py(i): return i+1 $$ diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index fc539647b63dd..afbb4a612135a 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -446,7 +446,7 @@ impl<'a> Binder { Plan::CreateFileFormat(Box::new(CreateFileFormatPlan { create_option: *create_option, name: name.clone(), - file_format_params: file_format_options.clone().try_into()?, + file_format_params: file_format_options.to_meta_ast().try_into()?, })) } Statement::DropFileFormat { diff --git a/src/query/sql/src/planner/binder/ddl/stage.rs b/src/query/sql/src/planner/binder/ddl/stage.rs index 378658775dbe0..0d5ecfdc7b3b7 100644 --- a/src/query/sql/src/planner/binder/ddl/stage.rs +++ b/src/query/sql/src/planner/binder/ddl/stage.rs @@ -12,14 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::BTreeMap; use std::str::FromStr; use databend_common_ast::ast::CreateStageStmt; +use databend_common_ast::ast::FileFormatOptions; use databend_common_ast::ast::UriLocation; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::FileFormatParams; use databend_common_meta_app::principal::OnErrorMode; use databend_common_meta_app::principal::StageInfo; @@ -125,14 +124,13 @@ impl Binder { #[async_backtrace::framed] pub(crate) async fn try_resolve_file_format( &self, - options: &BTreeMap, + options: &FileFormatOptions, ) -> Result { - if let Some(name) = options.get("format_name") { + let options = options.to_meta_ast(); + if let Some(name) = options.options.get("format_name") { self.ctx.get_file_format(name).await } else { - FileFormatParams::try_from(FileFormatOptionsAst { - options: options.clone(), - }) + FileFormatParams::try_from(options) } } } diff --git a/src/query/sql/src/planner/binder/ddl/task.rs b/src/query/sql/src/planner/binder/ddl/task.rs index 1a19abd366309..ac3e8c5394fa3 100644 --- a/src/query/sql/src/planner/binder/ddl/task.rs +++ b/src/query/sql/src/planner/binder/ddl/task.rs @@ -129,7 +129,7 @@ impl Binder { schedule_opts: schedule_opts.clone(), suspend_task_after_num_failures: *suspend_task_after_num_failures, after: after.clone(), - when_condition: when_condition.clone(), + when_condition: when_condition.as_ref().map(|expr| expr.to_string()), comment: comments.clone(), session_parameters: session_parameters.clone(), error_integration: error_integration.clone(), diff --git a/src/query/sql/src/planner/binder/explain.rs b/src/query/sql/src/planner/binder/explain.rs index d5bd97c3d0567..7f67e837dc35d 100644 --- a/src/query/sql/src/planner/binder/explain.rs +++ b/src/query/sql/src/planner/binder/explain.rs @@ -47,13 +47,11 @@ impl ExplainConfigBuilder { pub fn add_option(mut self, option: &ExplainOption) -> Self { match option { - ExplainOption::Verbose(v) => self.verbose = *v, - ExplainOption::Logical(v) => self.logical = *v, - ExplainOption::Optimized(v) => { - if *v { - self.logical = true; - } - self.optimized = *v; + ExplainOption::Verbose => self.verbose = true, + ExplainOption::Logical => self.logical = true, + ExplainOption::Optimized => { + self.logical = true; + self.optimized = true; } } @@ -81,13 +79,13 @@ impl Binder { // Rewrite `EXPLAIN RAW` to `EXPLAIN(LOGICAL)` if matches!(kind, ExplainKind::Raw) { - builder = builder.add_option(&ExplainOption::Logical(true)); + builder = builder.add_option(&ExplainOption::Logical); } // Rewrite `EXPLAIN OPTIMIZED` to `EXPLAIN(LOGICAL, OPTIMIZED)` if matches!(kind, ExplainKind::Optimized) { - builder = builder.add_option(&ExplainOption::Logical(true)); - builder = builder.add_option(&ExplainOption::Optimized(true)); + builder = builder.add_option(&ExplainOption::Logical); + builder = builder.add_option(&ExplainOption::Optimized); } for option in options { diff --git a/src/query/sql/src/planner/binder/insert.rs b/src/query/sql/src/planner/binder/insert.rs index 43894447b6a0f..9e32cfef7dbc9 100644 --- a/src/query/sql/src/planner/binder/insert.rs +++ b/src/query/sql/src/planner/binder/insert.rs @@ -23,7 +23,6 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::TableSchema; use databend_common_expression::TableSchemaRefExt; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::OnErrorMode; use crate::binder::Binder; @@ -115,7 +114,7 @@ impl Binder { on_error_mode, start, } => { - let params = FileFormatOptionsAst { options: settings }.try_into()?; + let params = settings.to_meta_ast().try_into()?; Ok(InsertInputSource::StreamingWithFileFormat { format: params, start, diff --git a/src/query/sql/src/planner/binder/replace.rs b/src/query/sql/src/planner/binder/replace.rs index a8bf29acb1056..f3d6dd1c334e6 100644 --- a/src/query/sql/src/planner/binder/replace.rs +++ b/src/query/sql/src/planner/binder/replace.rs @@ -20,7 +20,6 @@ use databend_common_ast::ast::ReplaceStmt; use databend_common_ast::ast::Statement; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::OnErrorMode; use crate::binder::Binder; @@ -103,7 +102,7 @@ impl Binder { on_error_mode, start, } => { - let params = FileFormatOptionsAst { options: settings }.try_into()?; + let params = settings.to_meta_ast().try_into()?; Ok(InsertInputSource::StreamingWithFileFormat { format: params, start, diff --git a/src/query/sql/src/planner/plans/ddl/notification.rs b/src/query/sql/src/planner/plans/ddl/notification.rs index 0c3191410be59..013ba1b24bc34 100644 --- a/src/query/sql/src/planner/plans/ddl/notification.rs +++ b/src/query/sql/src/planner/plans/ddl/notification.rs @@ -71,7 +71,7 @@ pub struct CreateNotificationPlan { pub notification_type: NotificationType, pub enabled: bool, pub webhook_opts: Option, - pub comments: String, + pub comments: Option, } impl CreateNotificationPlan { diff --git a/src/query/sql/src/planner/plans/ddl/task.rs b/src/query/sql/src/planner/plans/ddl/task.rs index 30a455f3dc3a3..959f562fca881 100644 --- a/src/query/sql/src/planner/plans/ddl/task.rs +++ b/src/query/sql/src/planner/plans/ddl/task.rs @@ -90,7 +90,7 @@ pub struct CreateTaskPlan { pub error_integration: Option, pub session_parameters: BTreeMap, pub sql: TaskSql, - pub comment: String, + pub comment: Option, } impl CreateTaskPlan { From 03c204c7d0333e0fc0e7886337650c91875042db Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 15:59:07 +0800 Subject: [PATCH 02/10] fix --- .../service/src/interpreters/interpreter_notification_create.rs | 2 +- src/query/service/src/interpreters/interpreter_task_alter.rs | 2 +- src/query/service/src/interpreters/interpreter_task_create.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/service/src/interpreters/interpreter_notification_create.rs b/src/query/service/src/interpreters/interpreter_notification_create.rs index 76f8952790659..a3208aa897adf 100644 --- a/src/query/service/src/interpreters/interpreter_notification_create.rs +++ b/src/query/service/src/interpreters/interpreter_notification_create.rs @@ -51,7 +51,7 @@ impl CreateNotificationInterpreter { if_not_exists: plan.if_not_exists, notification_type: tp as i32, enabled: plan.enabled, - comments: Some(plan.comments), + comments: plan.comments, webhook_url: plan .webhook_opts .as_ref() diff --git a/src/query/service/src/interpreters/interpreter_task_alter.rs b/src/query/service/src/interpreters/interpreter_task_alter.rs index 34c12d1a3d1bc..df3ab67f0f87b 100644 --- a/src/query/service/src/interpreters/interpreter_task_alter.rs +++ b/src/query/service/src/interpreters/interpreter_task_alter.rs @@ -132,7 +132,7 @@ impl AlterTaskInterpreter { } AlterTaskOptions::ModifyWhen(sql) => { req.alter_task_type = AlterTaskType::ModifyWhen as i32; - req.when_condition = Some(sql); + req.when_condition = Some(sql.to_string()); } } req diff --git a/src/query/service/src/interpreters/interpreter_task_create.rs b/src/query/service/src/interpreters/interpreter_task_create.rs index 0ffbbf786e48a..e6a33e89bd147 100644 --- a/src/query/service/src/interpreters/interpreter_task_create.rs +++ b/src/query/service/src/interpreters/interpreter_task_create.rs @@ -58,7 +58,7 @@ impl CreateTaskInterpreter { tenant_id: plan.tenant, query_text: "".to_string(), owner, - comment: Some(plan.comment), + comment: plan.comment, schedule_options: plan.schedule_opts.map(make_schedule_options), warehouse_options: Some(make_warehouse_options(plan.warehouse_opts)), error_integration: plan.error_integration, From d3511ceedfa6e3de69800598ae7ec14d0793e157 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 16:01:34 +0800 Subject: [PATCH 03/10] fix --- src/query/ast/src/parser/statement.rs | 46 +++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index de4d7bb9f4bb9..2820992eacf85 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -3859,17 +3859,17 @@ pub fn merge_update_expr(i: Input) -> IResult { pub fn password_set_options(i: Input) -> IResult { map( rule! { - ( PASSWORD_MIN_LENGTH ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MAX_LENGTH ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MIN_UPPER_CASE_CHARS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MIN_LOWER_CASE_CHARS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MIN_NUMERIC_CHARS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MIN_SPECIAL_CHARS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MIN_AGE_DAYS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MAX_AGE_DAYS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_MAX_RETRIES ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_LOCKOUT_TIME_MINS ~ Eq ~ ^#literal_u64 ) ? - ~ ( PASSWORD_HISTORY ~ Eq ~ ^#literal_u64 ) ? + ( PASSWORD_MIN_LENGTH ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MAX_LENGTH ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MIN_UPPER_CASE_CHARS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MIN_LOWER_CASE_CHARS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MIN_NUMERIC_CHARS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MIN_SPECIAL_CHARS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MIN_AGE_DAYS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MAX_AGE_DAYS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_MAX_RETRIES ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_LOCKOUT_TIME_MINS ~ Eq ~ ^#literal_u64 )? + ~ ( PASSWORD_HISTORY ~ Eq ~ ^#literal_u64 )? ~ ( COMMENT ~ Eq ~ ^#literal_string)? }, |( @@ -3907,18 +3907,18 @@ pub fn password_set_options(i: Input) -> IResult { pub fn password_unset_options(i: Input) -> IResult { map( rule! { - PASSWORD_MIN_LENGTH ? - ~ PASSWORD_MAX_LENGTH ? - ~ PASSWORD_MIN_UPPER_CASE_CHARS ? - ~ PASSWORD_MIN_LOWER_CASE_CHARS ? - ~ PASSWORD_MIN_NUMERIC_CHARS ? - ~ PASSWORD_MIN_SPECIAL_CHARS ? - ~ PASSWORD_MIN_AGE_DAYS ? - ~ PASSWORD_MAX_AGE_DAYS ? - ~ PASSWORD_MAX_RETRIES ? - ~ PASSWORD_LOCKOUT_TIME_MINS ? - ~ PASSWORD_HISTORY ? - ~ COMMENT ? + PASSWORD_MIN_LENGTH? + ~ PASSWORD_MAX_LENGTH? + ~ PASSWORD_MIN_UPPER_CASE_CHARS? + ~ PASSWORD_MIN_LOWER_CASE_CHARS? + ~ PASSWORD_MIN_NUMERIC_CHARS? + ~ PASSWORD_MIN_SPECIAL_CHARS? + ~ PASSWORD_MIN_AGE_DAYS? + ~ PASSWORD_MAX_AGE_DAYS? + ~ PASSWORD_MAX_RETRIES? + ~ PASSWORD_LOCKOUT_TIME_MINS? + ~ PASSWORD_HISTORY? + ~ COMMENT? }, |( opt_min_length, From 91e7c12580f2db07b82cbbb68f46e76bd63507f0 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 19:16:41 +0800 Subject: [PATCH 04/10] fix --- scripts/ci/ci-run-sqlsmith-tests.sh | 0 src/query/ast/src/ast/common.rs | 14 +++ src/query/ast/src/ast/statements/statement.rs | 10 --- src/query/ast/src/ast/statements/table.rs | 7 +- src/query/ast/src/parser/parser.rs | 28 ++++++ src/query/ast/tests/it/parser.rs | 2 +- src/query/ast/tests/it/testdata/statement.txt | 87 ++++++++++++++++++- 7 files changed, 135 insertions(+), 13 deletions(-) mode change 100644 => 100755 scripts/ci/ci-run-sqlsmith-tests.sh diff --git a/scripts/ci/ci-run-sqlsmith-tests.sh b/scripts/ci/ci-run-sqlsmith-tests.sh old mode 100644 new mode 100755 diff --git a/src/query/ast/src/ast/common.rs b/src/query/ast/src/ast/common.rs index 80400566af703..f95357d3461c5 100644 --- a/src/query/ast/src/ast/common.rs +++ b/src/query/ast/src/ast/common.rs @@ -256,3 +256,17 @@ pub(crate) fn write_comma_separated_string_map( } Ok(()) } + +/// Write input map items into `field_a='x' field_b='y'` +pub(crate) fn write_space_separated_string_map( + f: &mut Formatter<'_>, + items: impl IntoIterator, +) -> std::fmt::Result { + for (i, (k, v)) in items.into_iter().enumerate() { + if i > 0 { + write!(f, " ")?; + } + write!(f, "{k} = '{v}'")?; + } + Ok(()) +} diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 4952ededc175b..530842f302969 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -367,16 +367,6 @@ impl Statement { _ => format!("{}", self), } } - - pub fn assert_idempotent_parser(&self) { - dbg!(&self); - let sql = self.to_string(); - let tokens = crate::parser::tokenize_sql(&sql).unwrap(); - let (parsed, _) = - crate::parser::parse_sql(&tokens, crate::parser::Dialect::PostgreSQL).unwrap(); - let reparsed = parsed.to_string(); - assert_eq!(sql, reparsed); - } } impl Display for Statement { diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index 298ab465b2b0d..c114934f0bf0e 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -25,6 +25,7 @@ use crate::ast::statements::show::ShowLimit; use crate::ast::write_comma_separated_list; use crate::ast::write_comma_separated_string_map; use crate::ast::write_dot_separated_list; +use crate::ast::write_space_separated_string_map; use crate::ast::Expr; use crate::ast::Identifier; use crate::ast::Query; @@ -187,7 +188,11 @@ impl Display for CreateTableStmt { } // Format table options - write_comma_separated_string_map(f, &self.table_options)?; + if !self.table_options.is_empty() { + write!(f, " ")?; + write_space_separated_string_map(f, &self.table_options)?; + } + if let Some(as_query) = &self.as_query { write!(f, " AS {as_query}")?; } diff --git a/src/query/ast/src/parser/parser.rs b/src/query/ast/src/parser/parser.rs index a3e30e8569d66..3af131d0d03dd 100644 --- a/src/query/ast/src/parser/parser.rs +++ b/src/query/ast/src/parser/parser.rs @@ -45,6 +45,34 @@ pub fn tokenize_sql(sql: &str) -> Result> { #[minitrace::trace] pub fn parse_sql(tokens: &[Token], dialect: Dialect) -> Result<(Statement, Option)> { let stmt = run_parser(tokens, dialect, ParseMode::Default, false, statement)?; + + // #[cfg(debug_assertions)] + // { + // // Check that the statement can be displayed and reparsed without loss + // let reparse_sql = stmt.stmt.to_string(); + // let reparse_tokens = crate::parser::tokenize_sql(&reparse_sql).unwrap(); + // let reparsed = run_parser( + // &reparse_tokens, + // Dialect::PostgreSQL, + // ParseMode::Default, + // false, + // statement, + // ); + // match reparsed { + // Ok(reparsed) => { + // let reparsed_sql = reparsed.stmt.to_string(); + // assert_eq!(reparse_sql, reparsed_sql); + // } + // Err(e) => { + // let original_sql = tokens[0].source.to_string(); + // panic!( + // "Failed to reparse SQL:\n{}\nAST:\n{:#?}\n{}", + // original_sql, stmt.stmt, e + // ); + // } + // } + // } + Ok((stmt.stmt, stmt.format)) } diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index d4e1e9953e84d..49f7614241d9a 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -144,6 +144,7 @@ fn test_statement() { r#"create database ctl.t engine = Default;"#, r#"create database t engine = Default;"#, r#"create database t FROM SHARE a.s;"#, + r#"CREATE TABLE `t3`(a int not null, b int not null, c int not null) bloom_index_columns='a,b,c' COMPRESSION='zstd' STORAGE_FORMAT='native';"#, r#"create or replace database a;"#, r#"drop database ctl.t;"#, r#"drop database if exists t;"#, @@ -658,7 +659,6 @@ $$;"#, for case in cases { let tokens = tokenize_sql(case).unwrap(); let (stmt, fmt) = parse_sql(&tokens, Dialect::PostgreSQL).unwrap(); - stmt.assert_idempotent_parser(); writeln!(file, "---------- Input ----------").unwrap(); writeln!(file, "{}", case).unwrap(); writeln!(file, "---------- Output ---------").unwrap(); diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index c423e0dd655fb..71900efdf27bf 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -2573,6 +2573,91 @@ CreateDatabase( ) +---------- Input ---------- +CREATE TABLE `t3`(a int not null, b int not null, c int not null) bloom_index_columns='a,b,c' COMPRESSION='zstd' STORAGE_FORMAT='native'; +---------- Output --------- +CREATE TABLE `t3` (a Int32 NOT NULL, b Int32 NOT NULL, c Int32 NOT NULL) bloom_index_columns = 'a,b,c' compression = 'zstd' storage_format = 'native' +---------- AST ------------ +CreateTable( + CreateTableStmt { + create_option: Create, + catalog: None, + database: None, + table: Identifier { + span: Some( + 13..17, + ), + name: "t3", + quote: Some( + '`', + ), + is_hole: false, + }, + source: Some( + Columns( + [ + ColumnDefinition { + name: Identifier { + span: Some( + 18..19, + ), + name: "a", + quote: None, + is_hole: false, + }, + data_type: NotNull( + Int32, + ), + expr: None, + comment: None, + }, + ColumnDefinition { + name: Identifier { + span: Some( + 34..35, + ), + name: "b", + quote: None, + is_hole: false, + }, + data_type: NotNull( + Int32, + ), + expr: None, + comment: None, + }, + ColumnDefinition { + name: Identifier { + span: Some( + 50..51, + ), + name: "c", + quote: None, + is_hole: false, + }, + data_type: NotNull( + Int32, + ), + expr: None, + comment: None, + }, + ], + ), + ), + engine: None, + uri_location: None, + cluster_by: [], + table_options: { + "bloom_index_columns": "a,b,c", + "compression": "zstd", + "storage_format": "native", + }, + as_query: None, + transient: false, + }, +) + + ---------- Input ---------- create or replace database a; ---------- Output --------- @@ -10560,7 +10645,7 @@ VacuumDropTable( ---------- Input ---------- CREATE TABLE t (a INT COMMENT 'col comment') COMMENT='table comment'; ---------- Output --------- -CREATE TABLE t (a Int32 COMMENT 'col comment')comment = 'table comment' +CREATE TABLE t (a Int32 COMMENT 'col comment') comment = 'table comment' ---------- AST ------------ CreateTable( CreateTableStmt { From c88218d20428819bd663f1e940cc2b6ccdad00e4 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 19:22:36 +0800 Subject: [PATCH 05/10] fix --- Cargo.lock | 1 - src/query/ast/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d3f82406c268..341c121ed49d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2738,7 +2738,6 @@ dependencies = [ "pretty", "pretty_assertions", "regex", - "serde_json", "strsim 0.10.0", "strum 0.24.1", "strum_macros 0.24.3", diff --git a/src/query/ast/Cargo.toml b/src/query/ast/Cargo.toml index 568367a3ca030..524303bf0e94b 100644 --- a/src/query/ast/Cargo.toml +++ b/src/query/ast/Cargo.toml @@ -30,7 +30,6 @@ nom-rule = "0.3.0" ordered-float = { workspace = true } pratt = "0.4.0" pretty = "0.11.3" -serde_json = { workspace = true } strsim = "0.10" strum = "0.24" strum_macros = "0.24" From 4f98096e71039af6f60675cb83cae94676a9f021 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 20:19:59 +0800 Subject: [PATCH 06/10] fix --- src/query/ast/src/ast/expr.rs | 1636 ++++++++--------- src/query/ast/src/ast/format/syntax/dml.rs | 6 +- src/query/ast/src/ast/query.rs | 6 + src/query/ast/src/ast/statements/statement.rs | 2 +- src/query/ast/src/parser/parser.rs | 2 +- src/query/ast/src/parser/query.rs | 19 +- src/query/ast/tests/it/parser.rs | 3 + src/query/ast/tests/it/testdata/expr.txt | 18 +- src/query/ast/tests/it/testdata/query.txt | 4 +- src/query/ast/tests/it/testdata/statement.txt | 305 +++ 10 files changed, 1152 insertions(+), 849 deletions(-) diff --git a/src/query/ast/src/ast/expr.rs b/src/query/ast/src/ast/expr.rs index 172808011ed42..b45870c095fa8 100644 --- a/src/query/ast/src/ast/expr.rs +++ b/src/query/ast/src/ast/expr.rs @@ -265,340 +265,6 @@ pub enum Expr { }, } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum IntervalKind { - Year, - Quarter, - Month, - Day, - Hour, - Minute, - Second, - Doy, - Week, - Dow, -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum SubqueryModifier { - Any, - All, - Some, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum Literal { - UInt64(#[drive(skip)] u64), - Float64(#[drive(skip)] f64), - Decimal256 { - #[drive(skip)] - value: i256, - #[drive(skip)] - precision: u8, - #[drive(skip)] - scale: u8, - }, - // Quoted string literal value - String(#[drive(skip)] String), - Boolean(#[drive(skip)] bool), - Null, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct FunctionCall { - /// Set to true if the function is aggregate function with `DISTINCT`, like `COUNT(DISTINCT a)` - #[drive(skip)] - pub distinct: bool, - pub name: Identifier, - pub args: Vec, - pub params: Vec, - pub window: Option, - pub lambda: Option, -} - -/// The display style for a map access expression -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum MapAccessor { - /// `[0][1]` - Bracket { key: Box }, - /// `.1` - DotNumber { - #[drive(skip)] - key: u64, - }, - /// `:a:b` - Colon { key: Identifier }, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum TypeName { - Boolean, - UInt8, - UInt16, - UInt32, - UInt64, - Int8, - Int16, - Int32, - Int64, - Float32, - Float64, - Decimal { - #[drive(skip)] - precision: u8, - #[drive(skip)] - scale: u8, - }, - Date, - Timestamp, - Binary, - String, - Array(Box), - Map { - key_type: Box, - val_type: Box, - }, - Bitmap, - Tuple { - #[drive(skip)] - fields_name: Option>, - fields_type: Vec, - }, - Variant, - Geometry, - Nullable(Box), - NotNull(Box), -} - -impl TypeName { - pub fn is_nullable(&self) -> bool { - matches!(self, TypeName::Nullable(_)) - } - - pub fn wrap_nullable(self) -> Self { - if !self.is_nullable() { - Self::Nullable(Box::new(self)) - } else { - self - } - } - - pub fn wrap_not_null(self) -> Self { - match self { - Self::NotNull(_) => self, - _ => Self::NotNull(Box::new(self)), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum TrimWhere { - Both, - Leading, - Trailing, -} - -#[derive(Debug, Clone, PartialEq, EnumAsInner, Drive, DriveMut)] -pub enum Window { - WindowReference(WindowRef), - WindowSpec(WindowSpec), -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct WindowDefinition { - pub name: Identifier, - pub spec: WindowSpec, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct WindowRef { - pub window_name: Identifier, -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct WindowSpec { - pub existing_window_name: Option, - pub partition_by: Vec, - pub order_by: Vec, - pub window_frame: Option, -} - -/// `RANGE UNBOUNDED PRECEDING` or `ROWS BETWEEN 5 PRECEDING AND CURRENT ROW`. -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct WindowFrame { - pub units: WindowFrameUnits, - pub start_bound: WindowFrameBound, - pub end_bound: WindowFrameBound, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, EnumAsInner, Drive, DriveMut)] -pub enum WindowFrameUnits { - Rows, - Range, -} - -/// Specifies [WindowFrame]'s `start_bound` and `end_bound` -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub enum WindowFrameBound { - /// `CURRENT ROW` - CurrentRow, - /// ` PRECEDING` or `UNBOUNDED PRECEDING` - Preceding(Option>), - /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. - Following(Option>), -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct Lambda { - pub params: Vec, - pub expr: Box, -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum BinaryOperator { - Plus, - Minus, - Multiply, - Div, - Divide, - IntDiv, - Modulo, - StringConcat, - // `>` operator - Gt, - // `<` operator - Lt, - // `>=` operator - Gte, - // `<=` operator - Lte, - Eq, - NotEq, - Caret, - And, - Or, - Xor, - Like, - NotLike, - Regexp, - RLike, - NotRegexp, - NotRLike, - SoundsLike, - BitwiseOr, - BitwiseAnd, - BitwiseXor, - BitwiseShiftLeft, - BitwiseShiftRight, - L2Distance, -} - -impl BinaryOperator { - pub fn to_contrary(&self) -> Result { - match &self { - BinaryOperator::Gt => Ok(BinaryOperator::Lte), - BinaryOperator::Lt => Ok(BinaryOperator::Gte), - BinaryOperator::Gte => Ok(BinaryOperator::Lt), - BinaryOperator::Lte => Ok(BinaryOperator::Gt), - BinaryOperator::Eq => Ok(BinaryOperator::NotEq), - BinaryOperator::NotEq => Ok(BinaryOperator::Eq), - _ => Err(ErrorCode::Unimplemented(format!( - "Converting {self} to its contrary is not currently supported" - ))), - } - } - - pub fn to_func_name(&self) -> String { - match self { - BinaryOperator::StringConcat => "concat".to_string(), - BinaryOperator::BitwiseOr => "bit_or".to_string(), - BinaryOperator::BitwiseAnd => "bit_and".to_string(), - BinaryOperator::BitwiseXor => "bit_xor".to_string(), - BinaryOperator::BitwiseShiftLeft => "bit_shift_left".to_string(), - BinaryOperator::BitwiseShiftRight => "bit_shift_right".to_string(), - BinaryOperator::Caret => "pow".to_string(), - BinaryOperator::L2Distance => "l2_distance".to_string(), - _ => { - let name = format!("{:?}", self); - name.to_lowercase() - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum JsonOperator { - /// -> keeps the value as json - Arrow, - /// ->> keeps the value as text or int. - LongArrow, - /// #> Extracts JSON sub-object at the specified path - HashArrow, - /// #>> Extracts JSON sub-object at the specified path as text - HashLongArrow, - /// ? Checks whether text key exist as top-level key or array element. - Question, - /// ?| Checks whether any of the text keys exist as top-level keys or array elements. - QuestionOr, - /// ?& Checks whether all of the text keys exist as top-level keys or array elements. - QuestionAnd, - /// @> Checks whether left json contains the right json - AtArrow, - /// <@ Checks whether right json contains the left json - ArrowAt, - /// @? Checks whether JSON path return any item for the specified JSON value - AtQuestion, - /// @@ Returns the result of a JSON path predicate check for the specified JSON value. - AtAt, - /// #- Deletes the field or array element at the specified keypath. - HashMinus, -} - -impl JsonOperator { - pub fn to_func_name(&self) -> String { - match self { - JsonOperator::Arrow => "get".to_string(), - JsonOperator::LongArrow => "get_string".to_string(), - JsonOperator::HashArrow => "get_by_keypath".to_string(), - JsonOperator::HashLongArrow => "get_by_keypath_string".to_string(), - JsonOperator::Question => "json_exists_key".to_string(), - JsonOperator::QuestionOr => "json_exists_any_keys".to_string(), - JsonOperator::QuestionAnd => "json_exists_all_keys".to_string(), - JsonOperator::AtArrow => "json_contains_in_left".to_string(), - JsonOperator::ArrowAt => "json_contains_in_right".to_string(), - JsonOperator::AtQuestion => "json_path_exists".to_string(), - JsonOperator::AtAt => "json_path_match".to_string(), - JsonOperator::HashMinus => "delete_by_keypath".to_string(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] -pub enum UnaryOperator { - Plus, - Minus, - Not, - Factorial, - SquareRoot, - CubeRoot, - Abs, - BitwiseNot, -} - -impl UnaryOperator { - pub fn to_func_name(&self) -> String { - match self { - UnaryOperator::SquareRoot => "sqrt".to_string(), - UnaryOperator::CubeRoot => "cbrt".to_string(), - UnaryOperator::BitwiseNot => "bit_not".to_string(), - _ => { - let name = format!("{:?}", self); - name.to_lowercase() - } - } - } -} - impl Expr { pub fn span(&self) -> Span { match self { @@ -634,24 +300,284 @@ impl Expr { | Expr::DateTrunc { span, .. } | Expr::Hole { span, .. } => *span, } - } + } + + pub fn all_function_like_syntaxes() -> &'static [&'static str] { + &[ + "CAST", + "TRY_CAST", + "EXTRACT", + "DATE_PART", + "POSITION", + "SUBSTRING", + "TRIM", + "DATE_ADD", + "DATE_SUB", + "DATE_TRUNC", + ] + } +} + +impl Display for Expr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Expr::ColumnRef { column, .. } => { + if f.alternate() { + write!(f, "{column:#}")?; + } else { + write!(f, "{column}")?; + } + } + Expr::IsNull { expr, not, .. } => { + write!(f, "{expr} IS")?; + if *not { + write!(f, " NOT")?; + } + write!(f, " NULL")?; + } + Expr::IsDistinctFrom { + left, right, not, .. + } => { + write!(f, "{left} IS")?; + if *not { + write!(f, " NOT")?; + } + write!(f, " DISTINCT FROM {right}")?; + } + + Expr::InList { + expr, list, not, .. + } => { + write!(f, "{expr}")?; + if *not { + write!(f, " NOT")?; + } + write!(f, " IN(")?; + write_comma_separated_list(f, list)?; + write!(f, ")")?; + } + Expr::InSubquery { + expr, + subquery, + not, + .. + } => { + write!(f, "{expr}")?; + if *not { + write!(f, " NOT")?; + } + write!(f, " IN({subquery})")?; + } + Expr::Between { + expr, + low, + high, + not, + .. + } => { + write!(f, "{expr}")?; + if *not { + write!(f, " NOT")?; + } + write!(f, " BETWEEN {low} AND {high}")?; + } + Expr::UnaryOp { op, expr, .. } => { + match op { + // TODO (xieqijun) Maybe special attribute are provided to check whether the symbol is before or after. + UnaryOperator::Factorial => { + write!(f, "({expr} {op})")?; + } + _ => { + write!(f, "({op} {expr})")?; + } + } + } + Expr::BinaryOp { + op, left, right, .. + } => { + write!(f, "({left} {op} {right})")?; + } + Expr::JsonOp { + op, left, right, .. + } => { + write!(f, "({left} {op} {right})")?; + } + Expr::Cast { + expr, + target_type, + pg_style, + .. + } => { + if *pg_style { + write!(f, "{expr}::{target_type}")?; + } else { + write!(f, "CAST({expr} AS {target_type})")?; + } + } + Expr::TryCast { + expr, target_type, .. + } => { + write!(f, "TRY_CAST({expr} AS {target_type})")?; + } + Expr::Extract { + kind: field, expr, .. + } => { + write!(f, "EXTRACT({field} FROM {expr})")?; + } + Expr::DatePart { + kind: field, expr, .. + } => { + write!(f, "DATE_PART({field}, {expr})")?; + } + Expr::Position { + substr_expr, + str_expr, + .. + } => { + write!(f, "POSITION({substr_expr} IN {str_expr})")?; + } + Expr::Substring { + expr, + substring_from, + substring_for, + .. + } => { + write!(f, "SUBSTRING({expr} FROM {substring_from}")?; + if let Some(substring_for) = substring_for { + write!(f, " FOR {substring_for}")?; + } + write!(f, ")")?; + } + Expr::Trim { + expr, trim_where, .. + } => { + write!(f, "TRIM(")?; + if let Some((trim_where, trim_str)) = trim_where { + write!(f, "{trim_where} {trim_str} FROM ")?; + } + write!(f, "{expr})")?; + } + Expr::Literal { lit, .. } => { + write!(f, "{lit}")?; + } + Expr::CountAll { window, .. } => { + write!(f, "COUNT(*)")?; + if let Some(window) = window { + write!(f, " OVER ({window})")?; + } + } + Expr::Tuple { exprs, .. } => { + write!(f, "(")?; + write_comma_separated_list(f, exprs)?; + if exprs.len() == 1 { + write!(f, ",")?; + } + write!(f, ")")?; + } + Expr::FunctionCall { func, .. } => { + write!(f, "{func}")?; + } + Expr::Case { + operand, + conditions, + results, + else_result, + .. + } => { + write!(f, "CASE")?; + if let Some(op) = operand { + write!(f, " {op} ")?; + } + for (cond, res) in conditions.iter().zip(results) { + write!(f, " WHEN {cond} THEN {res}")?; + } + if let Some(el) = else_result { + write!(f, " ELSE {el}")?; + } + write!(f, " END")?; + } + Expr::Exists { not, subquery, .. } => { + if *not { + write!(f, "NOT ")?; + } + write!(f, "EXISTS ({subquery})")?; + } + Expr::Subquery { + subquery, modifier, .. + } => { + if let Some(m) = modifier { + write!(f, "{m} ")?; + } + write!(f, "({subquery})")?; + } + Expr::MapAccess { expr, accessor, .. } => { + write!(f, "{}", expr)?; + match accessor { + MapAccessor::Bracket { key } => write!(f, "[{key}]")?, + MapAccessor::DotNumber { key } => write!(f, ".{key}")?, + MapAccessor::Colon { key } => write!(f, ":{key}")?, + } + } + Expr::Array { exprs, .. } => { + write!(f, "[")?; + write_comma_separated_list(f, exprs)?; + write!(f, "]")?; + } + Expr::Map { kvs, .. } => { + write!(f, "{{")?; + for (i, (k, v)) in kvs.iter().enumerate() { + if i > 0 { + write!(f, ",")?; + } + write!(f, "{k}:{v}")?; + } + write!(f, "}}")?; + } + Expr::Interval { expr, unit, .. } => { + write!(f, "INTERVAL {expr} {unit}")?; + } + Expr::DateAdd { + unit, + interval, + date, + .. + } => { + write!(f, "DATE_ADD({unit}, {interval}, {date})")?; + } + Expr::DateSub { + unit, + interval, + date, + .. + } => { + write!(f, "DATE_SUB({unit}, {interval}, {date})")?; + } + Expr::DateTrunc { unit, date, .. } => { + write!(f, "DATE_TRUNC({unit}, {date})")?; + } + Expr::Hole { name, .. } => { + write!(f, ":{name}")?; + } + } - pub fn all_function_like_syntaxes() -> &'static [&'static str] { - &[ - "CAST", - "TRY_CAST", - "EXTRACT", - "DATE_PART", - "POSITION", - "SUBSTRING", - "TRIM", - "DATE_ADD", - "DATE_SUB", - "DATE_TRUNC", - ] + Ok(()) } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum IntervalKind { + Year, + Quarter, + Month, + Day, + Hour, + Minute, + Second, + Doy, + Week, + Dow, +} + impl Display for IntervalKind { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.write_str(match self { @@ -669,37 +595,11 @@ impl Display for IntervalKind { } } -impl Display for FunctionCall { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let FunctionCall { - distinct, - name, - args, - params, - window, - lambda, - } = self; - write!(f, "{name}")?; - if !params.is_empty() { - write!(f, "(")?; - write_comma_separated_list(f, params)?; - write!(f, ")")?; - } - write!(f, "(")?; - if *distinct { - write!(f, "DISTINCT ")?; - } - write_comma_separated_list(f, args)?; - if let Some(lambda) = lambda { - write!(f, ", {lambda}")?; - } - write!(f, ")")?; - - if let Some(window) = window { - write!(f, " OVER ({window})")?; - } - Ok(()) - } +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum SubqueryModifier { + Any, + All, + Some, } impl Display for SubqueryModifier { @@ -712,180 +612,173 @@ impl Display for SubqueryModifier { } } -impl Display for UnaryOperator { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - UnaryOperator::Plus => { - write!(f, "+") - } - UnaryOperator::Minus => { - write!(f, "-") - } - UnaryOperator::Not => { - write!(f, "NOT") - } - UnaryOperator::SquareRoot => { - write!(f, "|/") - } - UnaryOperator::CubeRoot => { - write!(f, "||/") - } - UnaryOperator::Factorial => { - write!(f, "!") - } - UnaryOperator::Abs => { - write!(f, "@") - } - UnaryOperator::BitwiseNot => { - write!(f, "~") - } - } - } -} - -impl Display for BinaryOperator { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - BinaryOperator::Plus => { - write!(f, "+") - } - BinaryOperator::Minus => { - write!(f, "-") - } - BinaryOperator::Multiply => { - write!(f, "*") - } - BinaryOperator::Div => { - write!(f, "DIV") - } - BinaryOperator::Divide => { - write!(f, "/") - } - BinaryOperator::IntDiv => { - write!(f, "//") - } - BinaryOperator::Modulo => { - write!(f, "%") - } - BinaryOperator::StringConcat => { - write!(f, "||") - } - BinaryOperator::Gt => { - write!(f, ">") - } - BinaryOperator::Lt => { - write!(f, "<") - } - BinaryOperator::Gte => { - write!(f, ">=") - } - BinaryOperator::Lte => { - write!(f, "<=") - } - BinaryOperator::Eq => { - write!(f, "=") - } - BinaryOperator::NotEq => { - write!(f, "<>") - } - BinaryOperator::Caret => { - write!(f, "^") - } - BinaryOperator::And => { - write!(f, "AND") - } - BinaryOperator::Or => { - write!(f, "OR") - } - BinaryOperator::Xor => { - write!(f, "XOR") - } - BinaryOperator::Like => { - write!(f, "LIKE") - } - BinaryOperator::NotLike => { - write!(f, "NOT LIKE") - } - BinaryOperator::Regexp => { - write!(f, "REGEXP") - } - BinaryOperator::RLike => { - write!(f, "RLIKE") - } - BinaryOperator::NotRegexp => { - write!(f, "NOT REGEXP") - } - BinaryOperator::NotRLike => { - write!(f, "NOT RLIKE") - } - BinaryOperator::SoundsLike => { - write!(f, "SOUNDS LIKE") - } - BinaryOperator::BitwiseOr => { - write!(f, "|") - } - BinaryOperator::BitwiseAnd => { - write!(f, "&") - } - BinaryOperator::BitwiseXor => { - write!(f, "#") - } - BinaryOperator::BitwiseShiftLeft => { - write!(f, "<<") - } - BinaryOperator::BitwiseShiftRight => { - write!(f, ">>") - } - BinaryOperator::L2Distance => { - write!(f, "<->") - } - } - } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum Literal { + UInt64(#[drive(skip)] u64), + Float64(#[drive(skip)] f64), + Decimal256 { + #[drive(skip)] + value: i256, + #[drive(skip)] + precision: u8, + #[drive(skip)] + scale: u8, + }, + // Quoted string literal value + String(#[drive(skip)] String), + Boolean(#[drive(skip)] bool), + Null, } -impl Display for JsonOperator { +impl Display for Literal { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - JsonOperator::Arrow => { - write!(f, "->") - } - JsonOperator::LongArrow => { - write!(f, "->>") - } - JsonOperator::HashArrow => { - write!(f, "#>") - } - JsonOperator::HashLongArrow => { - write!(f, "#>>") - } - JsonOperator::Question => { - write!(f, "?") - } - JsonOperator::QuestionOr => { - write!(f, "?|") - } - JsonOperator::QuestionAnd => { - write!(f, "?&") + match self { + Literal::UInt64(val) => { + write!(f, "{val}") } - JsonOperator::AtArrow => { - write!(f, "@>") + Literal::Decimal256 { value, scale, .. } => { + write!(f, "{}", display_decimal_256(*value, *scale)) } - JsonOperator::ArrowAt => { - write!(f, "<@") + Literal::Float64(val) => { + write!(f, "{val}") } - JsonOperator::AtQuestion => { - write!(f, "@?") + Literal::String(val) => { + write!(f, "\'{}\'", escape_string_with_quote(val, Some('\''))) } - JsonOperator::AtAt => { - write!(f, "@@") + Literal::Boolean(val) => { + if *val { + write!(f, "TRUE") + } else { + write!(f, "FALSE") + } } - JsonOperator::HashMinus => { - write!(f, "#-") + Literal::Null => { + write!(f, "NULL") } } } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct FunctionCall { + /// Set to true if the function is aggregate function with `DISTINCT`, like `COUNT(DISTINCT a)` + #[drive(skip)] + pub distinct: bool, + pub name: Identifier, + pub args: Vec, + pub params: Vec, + pub window: Option, + pub lambda: Option, +} + +impl Display for FunctionCall { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let FunctionCall { + distinct, + name, + args, + params, + window, + lambda, + } = self; + write!(f, "{name}")?; + if !params.is_empty() { + write!(f, "(")?; + write_comma_separated_list(f, params)?; + write!(f, ")")?; + } + write!(f, "(")?; + if *distinct { + write!(f, "DISTINCT ")?; + } + write_comma_separated_list(f, args)?; + if let Some(lambda) = lambda { + write!(f, ", {lambda}")?; + } + write!(f, ")")?; + + if let Some(window) = window { + write!(f, " OVER {window}")?; + } + Ok(()) + } +} + +/// The display style for a map access expression +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum MapAccessor { + /// `[0][1]` + Bracket { key: Box }, + /// `.1` + DotNumber { + #[drive(skip)] + key: u64, + }, + /// `:a:b` + Colon { key: Identifier }, +} + +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum TypeName { + Boolean, + UInt8, + UInt16, + UInt32, + UInt64, + Int8, + Int16, + Int32, + Int64, + Float32, + Float64, + Decimal { + #[drive(skip)] + precision: u8, + #[drive(skip)] + scale: u8, + }, + Date, + Timestamp, + Binary, + String, + Array(Box), + Map { + key_type: Box, + val_type: Box, + }, + Bitmap, + Tuple { + #[drive(skip)] + fields_name: Option>, + fields_type: Vec, + }, + Variant, + Geometry, + Nullable(Box), + NotNull(Box), +} + +impl TypeName { + pub fn is_nullable(&self) -> bool { + matches!(self, TypeName::Nullable(_)) + } + + pub fn wrap_nullable(self) -> Self { + if !self.is_nullable() { + Self::Nullable(Box::new(self)) + } else { + self + } + } + + pub fn wrap_not_null(self) -> Self { + match self { + Self::NotNull(_) => self, + _ => Self::NotNull(Box::new(self)), + } + } +} + impl Display for TypeName { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { @@ -991,6 +884,13 @@ impl Display for TypeName { } } +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum TrimWhere { + Both, + Leading, + Trailing, +} + impl Display for TrimWhere { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { f.write_str(match self { @@ -1001,116 +901,129 @@ impl Display for TrimWhere { } } -impl Display for Literal { +#[derive(Debug, Clone, PartialEq, EnumAsInner, Drive, DriveMut)] +pub enum Window { + WindowReference(WindowRef), + WindowSpec(WindowSpec), +} + +impl Display for Window { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Literal::UInt64(val) => { - write!(f, "{val}") - } - Literal::Decimal256 { value, scale, .. } => { - write!(f, "{}", display_decimal_256(*value, *scale)) - } - Literal::Float64(val) => { - write!(f, "{val}") - } - Literal::String(val) => { - write!(f, "\'{}\'", escape_string_with_quote(val, Some('\''))) - } - Literal::Boolean(val) => { - if *val { - write!(f, "TRUE") - } else { - write!(f, "FALSE") - } - } - Literal::Null => { - write!(f, "NULL") - } + match *self { + Window::WindowReference(ref window_ref) => write!(f, "{}", window_ref), + Window::WindowSpec(ref window_spec) => write!(f, "{}", window_spec), } } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct WindowDefinition { + pub name: Identifier, + pub spec: WindowSpec, +} + impl Display for WindowDefinition { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "WINDOW {} {}", self.name, self.spec) + write!(f, "{} AS {}", self.name, self.spec) } } -impl Display for Window { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let window_fmt = match *self { - Window::WindowSpec(ref window_spec) => format!("{}", window_spec), - Window::WindowReference(ref window_ref) => format!("{}", window_ref), - }; - write!(f, "{}", window_fmt) - } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct WindowRef { + pub window_name: Identifier, } impl Display for WindowRef { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "WINDOW {}", self.window_name) + write!(f, "{}", self.window_name) } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct WindowSpec { + pub existing_window_name: Option, + pub partition_by: Vec, + pub order_by: Vec, + pub window_frame: Option, +} + impl Display for WindowSpec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut first = true; + write!(f, "(")?; + if let Some(existing_window_name) = &self.existing_window_name { + write!(f, " {existing_window_name}")?; + } + if !self.partition_by.is_empty() { - first = false; - write!(f, "PARTITION BY ")?; - for (i, p) in self.partition_by.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{p}")?; - } + write!(f, " PARTITION BY ")?; + write_comma_separated_list(f, &self.partition_by)?; } if !self.order_by.is_empty() { - if !first { - write!(f, " ")?; - } - first = false; - write!(f, "ORDER BY ")?; - for (i, o) in self.order_by.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{o}")?; - } + write!(f, " ORDER BY ")?; + write_comma_separated_list(f, &self.order_by)?; } if let Some(frame) = &self.window_frame { - if !first { - write!(f, " ")?; - } match frame.units { WindowFrameUnits::Rows => { - write!(f, "ROWS")?; + write!(f, " ROWS")?; } WindowFrameUnits::Range => { - write!(f, "RANGE")?; + write!(f, " RANGE")?; } } - let format_frame = |frame: &WindowFrameBound| -> String { - match frame { - WindowFrameBound::CurrentRow => "CURRENT ROW".to_string(), - WindowFrameBound::Preceding(None) => "UNBOUNDED PRECEDING".to_string(), - WindowFrameBound::Following(None) => "UNBOUNDED FOLLOWING".to_string(), - WindowFrameBound::Preceding(Some(n)) => format!("{} PRECEDING", n), - WindowFrameBound::Following(Some(n)) => format!("{} FOLLOWING", n), - } - }; - write!( - f, - " BETWEEN {} AND {}", - format_frame(&frame.start_bound), - format_frame(&frame.end_bound) - )? - } - Ok(()) - } + let format_frame = |frame: &WindowFrameBound| -> String { + match frame { + WindowFrameBound::CurrentRow => "CURRENT ROW".to_string(), + WindowFrameBound::Preceding(None) => "UNBOUNDED PRECEDING".to_string(), + WindowFrameBound::Following(None) => "UNBOUNDED FOLLOWING".to_string(), + WindowFrameBound::Preceding(Some(n)) => format!("{} PRECEDING", n), + WindowFrameBound::Following(Some(n)) => format!("{} FOLLOWING", n), + } + }; + write!( + f, + " BETWEEN {} AND {}", + format_frame(&frame.start_bound), + format_frame(&frame.end_bound) + )? + } + write!(f, " )")?; + Ok(()) + } +} + +/// `RANGE UNBOUNDED PRECEDING` or `ROWS BETWEEN 5 PRECEDING AND CURRENT ROW`. +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct WindowFrame { + pub units: WindowFrameUnits, + pub start_bound: WindowFrameBound, + pub end_bound: WindowFrameBound, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, EnumAsInner, Drive, DriveMut)] +pub enum WindowFrameUnits { + Rows, + Range, +} + +/// Specifies [WindowFrame]'s `start_bound` and `end_bound` +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub enum WindowFrameBound { + /// `CURRENT ROW` + CurrentRow, + /// ` PRECEDING` or `UNBOUNDED PRECEDING` + Preceding(Option>), + /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. + Following(Option>), +} + +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct Lambda { + pub params: Vec, + pub expr: Box, } impl Display for Lambda { @@ -1128,249 +1041,322 @@ impl Display for Lambda { } } -impl Display for Expr { +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum BinaryOperator { + Plus, + Minus, + Multiply, + Div, + Divide, + IntDiv, + Modulo, + StringConcat, + // `>` operator + Gt, + // `<` operator + Lt, + // `>=` operator + Gte, + // `<=` operator + Lte, + Eq, + NotEq, + Caret, + And, + Or, + Xor, + Like, + NotLike, + Regexp, + RLike, + NotRegexp, + NotRLike, + SoundsLike, + BitwiseOr, + BitwiseAnd, + BitwiseXor, + BitwiseShiftLeft, + BitwiseShiftRight, + L2Distance, +} + +impl BinaryOperator { + pub fn to_contrary(&self) -> Result { + match &self { + BinaryOperator::Gt => Ok(BinaryOperator::Lte), + BinaryOperator::Lt => Ok(BinaryOperator::Gte), + BinaryOperator::Gte => Ok(BinaryOperator::Lt), + BinaryOperator::Lte => Ok(BinaryOperator::Gt), + BinaryOperator::Eq => Ok(BinaryOperator::NotEq), + BinaryOperator::NotEq => Ok(BinaryOperator::Eq), + _ => Err(ErrorCode::Unimplemented(format!( + "Converting {self} to its contrary is not currently supported" + ))), + } + } + + pub fn to_func_name(&self) -> String { + match self { + BinaryOperator::StringConcat => "concat".to_string(), + BinaryOperator::BitwiseOr => "bit_or".to_string(), + BinaryOperator::BitwiseAnd => "bit_and".to_string(), + BinaryOperator::BitwiseXor => "bit_xor".to_string(), + BinaryOperator::BitwiseShiftLeft => "bit_shift_left".to_string(), + BinaryOperator::BitwiseShiftRight => "bit_shift_right".to_string(), + BinaryOperator::Caret => "pow".to_string(), + BinaryOperator::L2Distance => "l2_distance".to_string(), + _ => { + let name = format!("{:?}", self); + name.to_lowercase() + } + } + } +} + +impl Display for BinaryOperator { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Expr::ColumnRef { column, .. } => { - if f.alternate() { - write!(f, "{column:#}")?; - } else { - write!(f, "{column}")?; - } + BinaryOperator::Plus => { + write!(f, "+") } - Expr::IsNull { expr, not, .. } => { - write!(f, "{expr} IS")?; - if *not { - write!(f, " NOT")?; - } - write!(f, " NULL")?; + BinaryOperator::Minus => { + write!(f, "-") } - Expr::IsDistinctFrom { - left, right, not, .. - } => { - write!(f, "{left} IS")?; - if *not { - write!(f, " NOT")?; - } - write!(f, " DISTINCT FROM {right}")?; + BinaryOperator::Multiply => { + write!(f, "*") } - - Expr::InList { - expr, list, not, .. - } => { - write!(f, "{expr}")?; - if *not { - write!(f, " NOT")?; - } - write!(f, " IN(")?; - write_comma_separated_list(f, list)?; - write!(f, ")")?; + BinaryOperator::Div => { + write!(f, "DIV") } - Expr::InSubquery { - expr, - subquery, - not, - .. - } => { - write!(f, "{expr}")?; - if *not { - write!(f, " NOT")?; - } - write!(f, " IN({subquery})")?; + BinaryOperator::Divide => { + write!(f, "/") } - Expr::Between { - expr, - low, - high, - not, - .. - } => { - write!(f, "{expr}")?; - if *not { - write!(f, " NOT")?; - } - write!(f, " BETWEEN {low} AND {high}")?; + BinaryOperator::IntDiv => { + write!(f, "//") } - Expr::UnaryOp { op, expr, .. } => { - match op { - // TODO (xieqijun) Maybe special attribute are provided to check whether the symbol is before or after. - UnaryOperator::Factorial => { - write!(f, "({expr} {op})")?; - } - _ => { - write!(f, "({op} {expr})")?; - } - } + BinaryOperator::Modulo => { + write!(f, "%") } - Expr::BinaryOp { - op, left, right, .. - } => { - write!(f, "({left} {op} {right})")?; + BinaryOperator::StringConcat => { + write!(f, "||") } - Expr::JsonOp { - op, left, right, .. - } => { - write!(f, "({left} {op} {right})")?; + BinaryOperator::Gt => { + write!(f, ">") } - Expr::Cast { - expr, - target_type, - pg_style, - .. - } => { - if *pg_style { - write!(f, "{expr}::{target_type}")?; - } else { - write!(f, "CAST({expr} AS {target_type})")?; - } + BinaryOperator::Lt => { + write!(f, "<") + } + BinaryOperator::Gte => { + write!(f, ">=") + } + BinaryOperator::Lte => { + write!(f, "<=") + } + BinaryOperator::Eq => { + write!(f, "=") + } + BinaryOperator::NotEq => { + write!(f, "<>") + } + BinaryOperator::Caret => { + write!(f, "^") + } + BinaryOperator::And => { + write!(f, "AND") + } + BinaryOperator::Or => { + write!(f, "OR") + } + BinaryOperator::Xor => { + write!(f, "XOR") + } + BinaryOperator::Like => { + write!(f, "LIKE") + } + BinaryOperator::NotLike => { + write!(f, "NOT LIKE") + } + BinaryOperator::Regexp => { + write!(f, "REGEXP") + } + BinaryOperator::RLike => { + write!(f, "RLIKE") + } + BinaryOperator::NotRegexp => { + write!(f, "NOT REGEXP") + } + BinaryOperator::NotRLike => { + write!(f, "NOT RLIKE") + } + BinaryOperator::SoundsLike => { + write!(f, "SOUNDS LIKE") + } + BinaryOperator::BitwiseOr => { + write!(f, "|") + } + BinaryOperator::BitwiseAnd => { + write!(f, "&") + } + BinaryOperator::BitwiseXor => { + write!(f, "#") + } + BinaryOperator::BitwiseShiftLeft => { + write!(f, "<<") } - Expr::TryCast { - expr, target_type, .. - } => { - write!(f, "TRY_CAST({expr} AS {target_type})")?; + BinaryOperator::BitwiseShiftRight => { + write!(f, ">>") } - Expr::Extract { - kind: field, expr, .. - } => { - write!(f, "EXTRACT({field} FROM {expr})")?; + BinaryOperator::L2Distance => { + write!(f, "<->") } - Expr::DatePart { - kind: field, expr, .. - } => { - write!(f, "DATE_PART({field}, {expr})")?; + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum JsonOperator { + /// -> keeps the value as json + Arrow, + /// ->> keeps the value as text or int. + LongArrow, + /// #> Extracts JSON sub-object at the specified path + HashArrow, + /// #>> Extracts JSON sub-object at the specified path as text + HashLongArrow, + /// ? Checks whether text key exist as top-level key or array element. + Question, + /// ?| Checks whether any of the text keys exist as top-level keys or array elements. + QuestionOr, + /// ?& Checks whether all of the text keys exist as top-level keys or array elements. + QuestionAnd, + /// @> Checks whether left json contains the right json + AtArrow, + /// <@ Checks whether right json contains the left json + ArrowAt, + /// @? Checks whether JSON path return any item for the specified JSON value + AtQuestion, + /// @@ Returns the result of a JSON path predicate check for the specified JSON value. + AtAt, + /// #- Deletes the field or array element at the specified keypath. + HashMinus, +} + +impl JsonOperator { + pub fn to_func_name(&self) -> String { + match self { + JsonOperator::Arrow => "get".to_string(), + JsonOperator::LongArrow => "get_string".to_string(), + JsonOperator::HashArrow => "get_by_keypath".to_string(), + JsonOperator::HashLongArrow => "get_by_keypath_string".to_string(), + JsonOperator::Question => "json_exists_key".to_string(), + JsonOperator::QuestionOr => "json_exists_any_keys".to_string(), + JsonOperator::QuestionAnd => "json_exists_all_keys".to_string(), + JsonOperator::AtArrow => "json_contains_in_left".to_string(), + JsonOperator::ArrowAt => "json_contains_in_right".to_string(), + JsonOperator::AtQuestion => "json_path_exists".to_string(), + JsonOperator::AtAt => "json_path_match".to_string(), + JsonOperator::HashMinus => "delete_by_keypath".to_string(), + } + } +} + +impl Display for JsonOperator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + JsonOperator::Arrow => { + write!(f, "->") } - Expr::Position { - substr_expr, - str_expr, - .. - } => { - write!(f, "POSITION({substr_expr} IN {str_expr})")?; + JsonOperator::LongArrow => { + write!(f, "->>") } - Expr::Substring { - expr, - substring_from, - substring_for, - .. - } => { - write!(f, "SUBSTRING({expr} FROM {substring_from}")?; - if let Some(substring_for) = substring_for { - write!(f, " FOR {substring_for}")?; - } - write!(f, ")")?; + JsonOperator::HashArrow => { + write!(f, "#>") } - Expr::Trim { - expr, trim_where, .. - } => { - write!(f, "TRIM(")?; - if let Some((trim_where, trim_str)) = trim_where { - write!(f, "{trim_where} {trim_str} FROM ")?; - } - write!(f, "{expr})")?; + JsonOperator::HashLongArrow => { + write!(f, "#>>") } - Expr::Literal { lit, .. } => { - write!(f, "{lit}")?; + JsonOperator::Question => { + write!(f, "?") } - Expr::CountAll { window, .. } => { - write!(f, "COUNT(*)")?; - if let Some(window) = window { - write!(f, " OVER ({window})")?; - } + JsonOperator::QuestionOr => { + write!(f, "?|") } - Expr::Tuple { exprs, .. } => { - write!(f, "(")?; - write_comma_separated_list(f, exprs)?; - if exprs.len() == 1 { - write!(f, ",")?; - } - write!(f, ")")?; + JsonOperator::QuestionAnd => { + write!(f, "?&") } - Expr::FunctionCall { func, .. } => { - write!(f, "{func}")?; + JsonOperator::AtArrow => { + write!(f, "@>") } - Expr::Case { - operand, - conditions, - results, - else_result, - .. - } => { - write!(f, "CASE")?; - if let Some(op) = operand { - write!(f, " {op} ")?; - } - for (cond, res) in conditions.iter().zip(results) { - write!(f, " WHEN {cond} THEN {res}")?; - } - if let Some(el) = else_result { - write!(f, " ELSE {el}")?; - } - write!(f, " END")?; + JsonOperator::ArrowAt => { + write!(f, "<@") } - Expr::Exists { not, subquery, .. } => { - if *not { - write!(f, "NOT ")?; - } - write!(f, "EXISTS ({subquery})")?; + JsonOperator::AtQuestion => { + write!(f, "@?") } - Expr::Subquery { - subquery, modifier, .. - } => { - if let Some(m) = modifier { - write!(f, "{m} ")?; - } - write!(f, "({subquery})")?; + JsonOperator::AtAt => { + write!(f, "@@") } - Expr::MapAccess { expr, accessor, .. } => { - write!(f, "{}", expr)?; - match accessor { - MapAccessor::Bracket { key } => write!(f, "[{key}]")?, - MapAccessor::DotNumber { key } => write!(f, ".{key}")?, - MapAccessor::Colon { key } => write!(f, ":{key}")?, - } + JsonOperator::HashMinus => { + write!(f, "#-") } - Expr::Array { exprs, .. } => { - write!(f, "[")?; - write_comma_separated_list(f, exprs)?; - write!(f, "]")?; + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Drive, DriveMut)] +pub enum UnaryOperator { + Plus, + Minus, + Not, + Factorial, + SquareRoot, + CubeRoot, + Abs, + BitwiseNot, +} + +impl Display for UnaryOperator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + UnaryOperator::Plus => { + write!(f, "+") } - Expr::Map { kvs, .. } => { - write!(f, "{{")?; - for (i, (k, v)) in kvs.iter().enumerate() { - if i > 0 { - write!(f, ",")?; - } - write!(f, "{k}:{v}")?; - } - write!(f, "}}")?; + UnaryOperator::Minus => { + write!(f, "-") } - Expr::Interval { expr, unit, .. } => { - write!(f, "INTERVAL {expr} {unit}")?; + UnaryOperator::Not => { + write!(f, "NOT") } - Expr::DateAdd { - unit, - interval, - date, - .. - } => { - write!(f, "DATE_ADD({unit}, {interval}, {date})")?; + UnaryOperator::SquareRoot => { + write!(f, "|/") } - Expr::DateSub { - unit, - interval, - date, - .. - } => { - write!(f, "DATE_SUB({unit}, {interval}, {date})")?; + UnaryOperator::CubeRoot => { + write!(f, "||/") } - Expr::DateTrunc { unit, date, .. } => { - write!(f, "DATE_TRUNC({unit}, {date})")?; + UnaryOperator::Factorial => { + write!(f, "!") } - Expr::Hole { name, .. } => { - write!(f, ":{name}")?; + UnaryOperator::Abs => { + write!(f, "@") + } + UnaryOperator::BitwiseNot => { + write!(f, "~") } } + } +} - Ok(()) +impl UnaryOperator { + pub fn to_func_name(&self) -> String { + match self { + UnaryOperator::SquareRoot => "sqrt".to_string(), + UnaryOperator::CubeRoot => "cbrt".to_string(), + UnaryOperator::BitwiseNot => "bit_not".to_string(), + _ => { + let name = format!("{:?}", self); + name.to_lowercase() + } + } } } diff --git a/src/query/ast/src/ast/format/syntax/dml.rs b/src/query/ast/src/ast/format/syntax/dml.rs index 552781bd58473..d5c0dd6c836b8 100644 --- a/src/query/ast/src/ast/format/syntax/dml.rs +++ b/src/query/ast/src/ast/format/syntax/dml.rs @@ -99,7 +99,7 @@ fn pretty_source(source: InsertSource) -> RcDoc<'static> { .append(RcDoc::space()) .append(RcDoc::text("=")) .append(RcDoc::space()) - .append(RcDoc::text(format!("{:?}", v))) + .append(RcDoc::text(format!("{}", v))) })) .group(), )) @@ -215,7 +215,7 @@ pub(crate) fn pretty_copy_into_table(copy_stmt: CopyIntoTableStmt) -> RcDoc<'sta .append(if let Some(pattern) = ©_stmt.pattern { RcDoc::line() .append(RcDoc::text("PATTERN = ")) - .append(RcDoc::text(format!("{:?}", pattern))) + .append(RcDoc::text(format!("'{}'", pattern))) } else { RcDoc::nil() }) @@ -291,7 +291,7 @@ fn pretty_file_format(file_format: &FileFormatOptions) -> RcDoc<'static> { .append(RcDoc::space()) .append(RcDoc::text("=")) .append(RcDoc::space()) - .append(RcDoc::text(format!("{:?}", v))) + .append(RcDoc::text(format!("{}", v))) })) .group(), )) diff --git a/src/query/ast/src/ast/query.rs b/src/query/ast/src/ast/query.rs index b3c94a93853ca..4c6d98ec66246 100644 --- a/src/query/ast/src/ast/query.rs +++ b/src/query/ast/src/ast/query.rs @@ -224,6 +224,12 @@ impl Display for SelectStmt { write!(f, " HAVING {having}")?; } + // WINDOW clause + if let Some(windows) = &self.window_list { + write!(f, " WINDOW ")?; + write_comma_separated_list(f, windows)?; + } + Ok(()) } } diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 530842f302969..cb5719445bb5b 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -631,7 +631,7 @@ impl Display for Statement { } Statement::CreateStage(stmt) => write!(f, "{stmt}")?, Statement::RemoveStage { location, pattern } => { - write!(f, "REMOVE STAGE @{location}")?; + write!(f, "REMOVE @{location}")?; if !pattern.is_empty() { write!(f, " PATTERN = '{pattern}'")?; } diff --git a/src/query/ast/src/parser/parser.rs b/src/query/ast/src/parser/parser.rs index 3af131d0d03dd..7a5e8366420c0 100644 --- a/src/query/ast/src/parser/parser.rs +++ b/src/query/ast/src/parser/parser.rs @@ -61,7 +61,7 @@ pub fn parse_sql(tokens: &[Token], dialect: Dialect) -> Result<(Statement, Optio // match reparsed { // Ok(reparsed) => { // let reparsed_sql = reparsed.stmt.to_string(); - // assert_eq!(reparse_sql, reparsed_sql); + // assert_eq!(reparse_sql, reparsed_sql, "AST:\n{:#?}", stmt.stmt); // } // Err(e) => { // let original_sql = tokens[0].source.to_string(); diff --git a/src/query/ast/src/parser/query.rs b/src/query/ast/src/parser/query.rs index 9b976a5208d66..db6c7967b7e88 100644 --- a/src/query/ast/src/parser/query.rs +++ b/src/query/ast/src/parser/query.rs @@ -1002,7 +1002,7 @@ pub fn window_frame_between(i: Input) -> IResult<(WindowFrameBound, WindowFrameB pub fn window_spec(i: Input) -> IResult { map( rule! { - (#ident )? + #ident? ~ ( PARTITION ~ ^BY ~ ^#comma_separated_list1(subexpr(0)) )? ~ ( ORDER ~ ^BY ~ ^#comma_separated_list1(order_by_expr) )? ~ ( (ROWS | RANGE) ~ ^#window_frame_between )? @@ -1032,24 +1032,27 @@ pub fn window_spec_ident(i: Input) -> IResult { alt(( map( rule! { - ("(" ~ #window_spec ~ ")") + "(" ~ #window_spec ~ ")" }, |(_, spec, _)| Window::WindowSpec(spec), ), - map(rule! {#ident}, |window_name| { - Window::WindowReference(WindowRef { window_name }) - }), + map( + rule! { + #ident + }, + |window_name| Window::WindowReference(WindowRef { window_name }), + ), ))(i) } pub fn window_clause(i: Input) -> IResult { map( rule! { - #ident ~ (AS ~ "(" ~ #window_spec ~ ")") + #ident ~ AS ~ "(" ~ #window_spec ~ ")" }, - |(ident, window)| WindowDefinition { + |(ident, _, _, window, _)| WindowDefinition { name: ident, - spec: window.2, + spec: window, }, )(i) } diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index 49f7614241d9a..abe872ef39f21 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -635,6 +635,9 @@ fn test_statement() { "--各环节转各环节转各环节转各环节转各\n select 34343", "-- 96477300355 31379974136 3.074486292973661\nselect 34343", "-- xxxxx\n select 34343;", + "REMOVE @t;", + "SELECT sum(d) OVER (w) FROM e;", + "SELECT sum(d) OVER w FROM e WINDOW w AS (PARTITION BY f ORDER BY g);", "GRANT OWNERSHIP ON d20_0014.* TO ROLE 'd20_0015_owner';", "GRANT OWNERSHIP ON d20_0014.t TO ROLE 'd20_0015_owner';", "GRANT OWNERSHIP ON STAGE s1 TO ROLE 'd20_0015_owner';", diff --git a/src/query/ast/tests/it/testdata/expr.txt b/src/query/ast/tests/it/testdata/expr.txt index cc6f033658815..3ebab67ce3276 100644 --- a/src/query/ast/tests/it/testdata/expr.txt +++ b/src/query/ast/tests/it/testdata/expr.txt @@ -3490,7 +3490,7 @@ Map { ---------- Input ---------- ROW_NUMBER() OVER (ORDER BY salary DESC) ---------- Output --------- -ROW_NUMBER() OVER (ORDER BY salary DESC) +ROW_NUMBER() OVER ( ORDER BY salary DESC ) ---------- AST ------------ FunctionCall { span: Some( @@ -3552,7 +3552,7 @@ FunctionCall { ---------- Input ---------- SUM(salary) OVER () ---------- Output --------- -SUM(salary) OVER () +SUM(salary) OVER ( ) ---------- AST ------------ FunctionCall { span: Some( @@ -3608,7 +3608,7 @@ FunctionCall { ---------- Input ---------- AVG(salary) OVER (PARTITION BY department) ---------- Output --------- -AVG(salary) OVER (PARTITION BY department) +AVG(salary) OVER ( PARTITION BY department ) ---------- AST ------------ FunctionCall { span: Some( @@ -3684,7 +3684,7 @@ FunctionCall { ---------- Input ---------- SUM(salary) OVER (PARTITION BY department ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) ---------- Output --------- -SUM(salary) OVER (PARTITION BY department ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) +SUM(salary) OVER ( PARTITION BY department ORDER BY salary DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( @@ -3794,7 +3794,7 @@ FunctionCall { ---------- Input ---------- AVG(salary) OVER (PARTITION BY department ORDER BY hire_date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) ---------- Output --------- -AVG(salary) OVER (PARTITION BY department ORDER BY hire_date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) +AVG(salary) OVER ( PARTITION BY department ORDER BY hire_date ROWS BETWEEN 2 PRECEDING AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( @@ -3911,7 +3911,7 @@ FunctionCall { ---------- Input ---------- COUNT() OVER (ORDER BY hire_date RANGE BETWEEN INTERVAL '7' DAY PRECEDING AND CURRENT ROW) ---------- Output --------- -COUNT() OVER (ORDER BY hire_date RANGE BETWEEN INTERVAL '7' DAY PRECEDING AND CURRENT ROW) +COUNT() OVER ( ORDER BY hire_date RANGE BETWEEN INTERVAL '7' DAY PRECEDING AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( @@ -3994,7 +3994,7 @@ FunctionCall { ---------- Input ---------- COUNT() OVER (ORDER BY hire_date ROWS UNBOUNDED PRECEDING) ---------- Output --------- -COUNT() OVER (ORDER BY hire_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) +COUNT() OVER ( ORDER BY hire_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( @@ -4062,7 +4062,7 @@ FunctionCall { ---------- Input ---------- COUNT() OVER (ORDER BY hire_date ROWS CURRENT ROW) ---------- Output --------- -COUNT() OVER (ORDER BY hire_date ROWS BETWEEN CURRENT ROW AND CURRENT ROW) +COUNT() OVER ( ORDER BY hire_date ROWS BETWEEN CURRENT ROW AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( @@ -4128,7 +4128,7 @@ FunctionCall { ---------- Input ---------- COUNT() OVER (ORDER BY hire_date ROWS 3 PRECEDING) ---------- Output --------- -COUNT() OVER (ORDER BY hire_date ROWS BETWEEN 3 PRECEDING AND CURRENT ROW) +COUNT() OVER ( ORDER BY hire_date ROWS BETWEEN 3 PRECEDING AND CURRENT ROW ) ---------- AST ------------ FunctionCall { span: Some( diff --git a/src/query/ast/tests/it/testdata/query.txt b/src/query/ast/tests/it/testdata/query.txt index 88b7cb2a46765..b4a1b755ec7f5 100644 --- a/src/query/ast/tests/it/testdata/query.txt +++ b/src/query/ast/tests/it/testdata/query.txt @@ -5552,7 +5552,7 @@ Query { ---------- Input ---------- select sum(a) over w from customer window w as (partition by a order by b) ---------- Output --------- -SELECT sum(a) OVER (WINDOW w) FROM customer +SELECT sum(a) OVER w FROM customer WINDOW w AS ( PARTITION BY a ORDER BY b ) ---------- AST ------------ Query { span: Some( @@ -5726,7 +5726,7 @@ Query { ---------- Input ---------- select a, sum(a) over w, sum(a) over w1, sum(a) over w2 from t1 window w as (partition by a), w2 as (w1 rows current row), w1 as (w order by a) order by a ---------- Output --------- -SELECT a, sum(a) OVER (WINDOW w), sum(a) OVER (WINDOW w1), sum(a) OVER (WINDOW w2) FROM t1 ORDER BY a +SELECT a, sum(a) OVER w, sum(a) OVER w1, sum(a) OVER w2 FROM t1 WINDOW w AS ( PARTITION BY a ), w2 AS ( w1 ROWS BETWEEN CURRENT ROW AND CURRENT ROW ), w1 AS ( w ORDER BY a ) ORDER BY a ---------- AST ------------ Query { span: Some( diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index 71900efdf27bf..202e9835b3f93 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -17767,6 +17767,311 @@ Query( ) +---------- Input ---------- +REMOVE @t; +---------- Output --------- +REMOVE @t +---------- AST ------------ +RemoveStage { + location: "t", + pattern: "", +} + + +---------- Input ---------- +SELECT sum(d) OVER (w) FROM e; +---------- Output --------- +SELECT sum(d) OVER ( w ) FROM e +---------- AST ------------ +Query( + Query { + span: Some( + 0..29, + ), + with: None, + body: Select( + SelectStmt { + span: Some( + 0..29, + ), + hints: None, + distinct: false, + select_list: [ + AliasedExpr { + expr: FunctionCall { + span: Some( + 7..22, + ), + func: FunctionCall { + distinct: false, + name: Identifier { + span: Some( + 7..10, + ), + name: "sum", + quote: None, + is_hole: false, + }, + args: [ + ColumnRef { + span: Some( + 11..12, + ), + column: ColumnRef { + database: None, + table: None, + column: Name( + Identifier { + span: Some( + 11..12, + ), + name: "d", + quote: None, + is_hole: false, + }, + ), + }, + }, + ], + params: [], + window: Some( + WindowSpec( + WindowSpec { + existing_window_name: Some( + Identifier { + span: Some( + 20..21, + ), + name: "w", + quote: None, + is_hole: false, + }, + ), + partition_by: [], + order_by: [], + window_frame: None, + }, + ), + ), + lambda: None, + }, + }, + alias: None, + }, + ], + from: [ + Table { + span: Some( + 28..29, + ), + catalog: None, + database: None, + table: Identifier { + span: Some( + 28..29, + ), + name: "e", + quote: None, + is_hole: false, + }, + alias: None, + travel_point: None, + since_point: None, + pivot: None, + unpivot: None, + }, + ], + selection: None, + group_by: None, + having: None, + window_list: None, + qualify: None, + }, + ), + order_by: [], + limit: [], + offset: None, + ignore_result: false, + }, +) + + +---------- Input ---------- +SELECT sum(d) OVER w FROM e WINDOW w AS (PARTITION BY f ORDER BY g); +---------- Output --------- +SELECT sum(d) OVER w FROM e WINDOW w AS ( PARTITION BY f ORDER BY g ) +---------- AST ------------ +Query( + Query { + span: Some( + 0..67, + ), + with: None, + body: Select( + SelectStmt { + span: Some( + 0..67, + ), + hints: None, + distinct: false, + select_list: [ + AliasedExpr { + expr: FunctionCall { + span: Some( + 7..20, + ), + func: FunctionCall { + distinct: false, + name: Identifier { + span: Some( + 7..10, + ), + name: "sum", + quote: None, + is_hole: false, + }, + args: [ + ColumnRef { + span: Some( + 11..12, + ), + column: ColumnRef { + database: None, + table: None, + column: Name( + Identifier { + span: Some( + 11..12, + ), + name: "d", + quote: None, + is_hole: false, + }, + ), + }, + }, + ], + params: [], + window: Some( + WindowReference( + WindowRef { + window_name: Identifier { + span: Some( + 19..20, + ), + name: "w", + quote: None, + is_hole: false, + }, + }, + ), + ), + lambda: None, + }, + }, + alias: None, + }, + ], + from: [ + Table { + span: Some( + 26..27, + ), + catalog: None, + database: None, + table: Identifier { + span: Some( + 26..27, + ), + name: "e", + quote: None, + is_hole: false, + }, + alias: None, + travel_point: None, + since_point: None, + pivot: None, + unpivot: None, + }, + ], + selection: None, + group_by: None, + having: None, + window_list: Some( + [ + WindowDefinition { + name: Identifier { + span: Some( + 35..36, + ), + name: "w", + quote: None, + is_hole: false, + }, + spec: WindowSpec { + existing_window_name: None, + partition_by: [ + ColumnRef { + span: Some( + 54..55, + ), + column: ColumnRef { + database: None, + table: None, + column: Name( + Identifier { + span: Some( + 54..55, + ), + name: "f", + quote: None, + is_hole: false, + }, + ), + }, + }, + ], + order_by: [ + OrderByExpr { + expr: ColumnRef { + span: Some( + 65..66, + ), + column: ColumnRef { + database: None, + table: None, + column: Name( + Identifier { + span: Some( + 65..66, + ), + name: "g", + quote: None, + is_hole: false, + }, + ), + }, + }, + asc: None, + nulls_first: None, + }, + ], + window_frame: None, + }, + }, + ], + ), + qualify: None, + }, + ), + order_by: [], + limit: [], + offset: None, + ignore_result: false, + }, +) + + ---------- Input ---------- GRANT OWNERSHIP ON d20_0014.* TO ROLE 'd20_0015_owner'; ---------- Output --------- From c680ec113fb6a7c874831964ccff39a2adb7249f Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 21:15:46 +0800 Subject: [PATCH 07/10] fix --- src/query/ast/src/ast/statements/copy.rs | 7 ++- src/query/ast/tests/it/testdata/statement.txt | 45 +++++++------------ .../mode/standalone/explain/explain.test | 24 +++++----- .../mode/standalone/explain/window.test | 30 ++++++------- 4 files changed, 47 insertions(+), 59 deletions(-) diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index d4eb70975b5e3..57d7ca6eadbc8 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -23,6 +23,7 @@ use std::str::FromStr; use databend_common_base::base::mask_string; use databend_common_exception::ErrorCode; +use databend_common_io::escape_string_with_quote; use databend_common_meta_app::principal::CopyOptions; use databend_common_meta_app::principal::FileFormatOptionsAst; use databend_common_meta_app::principal::OnErrorMode; @@ -531,14 +532,16 @@ impl Display for FileFormatValue { FileFormatValue::Keyword(v) => write!(f, "{v}"), FileFormatValue::Bool(v) => write!(f, "{v}"), FileFormatValue::U64(v) => write!(f, "{v}"), - FileFormatValue::String(v) => write!(f, "'{v}'"), + FileFormatValue::String(v) => { + write!(f, "'{}'", escape_string_with_quote(v, Some('\''))) + } FileFormatValue::StringList(v) => { write!(f, "(")?; for (i, s) in v.iter().enumerate() { if i > 0 { write!(f, ", ")?; } - write!(f, "'{s}'")?; + write!(f, "'{}'", escape_string_with_quote(s, Some('\'')))?; } write!(f, ")") } diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index 202e9835b3f93..30588e58d7968 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -11460,8 +11460,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM '@~/mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@~/mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11527,8 +11526,7 @@ COPY INTO mytable size_limit=10 max_files=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11603,8 +11601,7 @@ COPY INTO mytable size_limit=10 max_files=3000; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 3000 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 MAX_FILES = 3000 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11681,8 +11678,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11761,8 +11757,7 @@ COPY INTO mytable skip_header = 1 ); ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( endpoint_url = 'http://127.0.0.1:9900' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -11947,8 +11942,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM '@my_stage' FILE_FORMAT = (error_on_column_count_mismatch = false, field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@my_stage' FILE_FORMAT = (error_on_column_count_mismatch = false, field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12015,8 +12009,7 @@ COPY INTO 's3://mybucket/data.csv' skip_header = 1 ) ---------- Output --------- -COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +COPY INTO 's3://mybucket/data.csv' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CopyIntoLocation( CopyIntoLocationStmt { @@ -12116,8 +12109,7 @@ COPY INTO @my_stage skip_header = 1 ); ---------- Output --------- -COPY INTO '@my_stage' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false +COPY INTO '@my_stage' FROM mytable FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SINGLE = false MAX_FILE_SIZE = 0 DETAILED_OUTPUT = false ---------- AST ------------ CopyIntoLocation( CopyIntoLocationStmt { @@ -12177,8 +12169,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id = 'access_key', aws_secret_key = 'secret_key' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM 's3://mybucket/data.csv' CONNECTION = ( aws_key_id = 'access_key', aws_secret_key = 'secret_key' ) FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12255,8 +12246,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12321,8 +12311,7 @@ COPY INTO mytable ) size_limit=10; ---------- Output --------- -COPY INTO mytable FROM '@external_stage/path/to/dir/' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/dir/' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12387,8 +12376,7 @@ COPY INTO mytable ) force=true; ---------- Output --------- -COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = abort +COPY INTO mytable FROM '@external_stage/path/to/file.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) PURGE = false FORCE = true DISABLE_VARIANT_CHECK = false ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -12454,8 +12442,7 @@ COPY INTO mytable size_limit=10 disable_variant_check=true; ---------- Output --------- -COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = abort +COPY INTO mytable FROM 'fs:///path/to/data.csv' FILE_FORMAT = (field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV) SIZE_LIMIT = 10 PURGE = false FORCE = false DISABLE_VARIANT_CHECK = true ON_ERROR = abort ---------- AST ------------ CopyIntoTable( CopyIntoTableStmt { @@ -14623,8 +14610,7 @@ Query( CREATE FILE FORMAT my_csv type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1; ---------- Output --------- -CREATE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV +CREATE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV ---------- AST ------------ CreateFileFormat { create_option: Create, @@ -14652,8 +14638,7 @@ CreateFileFormat { CREATE OR REPLACE FILE FORMAT my_csv type = CSV field_delimiter = ',' record_delimiter = '\n' skip_header = 1; ---------- Output --------- -CREATE OR REPLACE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = ' -', skip_header = 1, type = CSV +CREATE OR REPLACE FILE FORMAT my_csv field_delimiter = ',', record_delimiter = '\n', skip_header = 1, type = CSV ---------- AST ------------ CreateFileFormat { create_option: CreateOrReplace, diff --git a/tests/sqllogictests/suites/mode/standalone/explain/explain.test b/tests/sqllogictests/suites/mode/standalone/explain/explain.test index a2a3ca00ed47b..82577b527cfe0 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/explain.test @@ -210,10 +210,10 @@ COPY INTO t1 FROM 's3://mybucket/data.csv' FILE_FORMAT = ( - field_delimiter = ",", - record_delimiter = "\n", - skip_header = "1", - type = "CSV" + field_delimiter = ',', + record_delimiter = '\n', + skip_header = 1, + type = CSV ) SIZE_LIMIT = 10 PURGE = false @@ -227,10 +227,10 @@ COPY INTO Uri(UriLocation { protocol: "s3", name: "mybucket", path: "/data.csv", part_prefix: "", connection: Connection { visited_keys: {}, conns: {} } }) FROM t1 FILE_FORMAT = ( - field_delimiter = ",", - record_delimiter = "\n", - skip_header = "1", - type = "CSV" + field_delimiter = ',', + record_delimiter = '\n', + skip_header = 1, + type = CSV ) SINGLE = false @@ -467,10 +467,10 @@ CopyIntoTable (children 7) ├── FROM (children 1) │ └── Location 's3://mybucket/data.csv' ├── FileFormats (children 4) -│ ├── FileFormat field_delimiter = "," -│ ├── FileFormat record_delimiter = "\n" -│ ├── FileFormat skip_header = "1" -│ └── FileFormat type = "CSV" +│ ├── FileFormat field_delimiter = String(",") +│ ├── FileFormat record_delimiter = String("\n") +│ ├── FileFormat skip_header = U64(1) +│ └── FileFormat type = Keyword("CSV") ├── SizeLimit 10 ├── MaxFiles 10 ├── Purge false diff --git a/tests/sqllogictests/suites/mode/standalone/explain/window.test b/tests/sqllogictests/suites/mode/standalone/explain/window.test index 3a7b435af4050..9c78d7b47fed2 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/window.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/window.test @@ -14,11 +14,11 @@ query T explain SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname ORDER BY empno) FROM empsalary ORDER BY depname, empno ---- Sort -├── output columns: [empsalary.depname (#0), empsalary.empno (#1), empsalary.salary (#2), sum(salary) OVER (PARTITION BY depname ORDER BY empno) (#4)] +├── output columns: [empsalary.depname (#0), empsalary.empno (#1), empsalary.salary (#2), sum(salary) OVER ( PARTITION BY depname ORDER BY empno ) (#4)] ├── sort keys: [depname ASC NULLS LAST, empno ASC NULLS LAST] ├── estimated rows: 0.00 └── Window - ├── output columns: [empsalary.depname (#0), empsalary.empno (#1), empsalary.salary (#2), sum(salary) OVER (PARTITION BY depname ORDER BY empno) (#4)] + ├── output columns: [empsalary.depname (#0), empsalary.empno (#1), empsalary.salary (#2), sum(salary) OVER ( PARTITION BY depname ORDER BY empno ) (#4)] ├── aggregate function: [sum(salary)] ├── partition by: [depname] ├── order by: [empno] @@ -95,7 +95,7 @@ Filter ├── filters: [t2.rank (#4) = 1] ├── estimated rows: 0.00 └── Window - ├── output columns: [test.k (#0), test.v (#1), rank() OVER (PARTITION BY k ORDER BY v DESC) (#4)] + ├── output columns: [test.k (#0), test.v (#1), rank() OVER ( PARTITION BY k ORDER BY v DESC ) (#4)] ├── aggregate function: [rank] ├── partition by: [k] ├── order by: [v] @@ -139,7 +139,7 @@ Filter ├── filters: [t2.rank (#4) = 1, is_true(t2.k (#0) = 12)] ├── estimated rows: 0.00 └── Window - ├── output columns: [test.k (#0), test.v (#1), rank() OVER (PARTITION BY v ORDER BY v DESC) (#4)] + ├── output columns: [test.k (#0), test.v (#1), rank() OVER ( PARTITION BY v ORDER BY v DESC ) (#4)] ├── aggregate function: [rank] ├── partition by: [v] ├── order by: [v] @@ -175,7 +175,7 @@ Filter ├── filters: [t2.rank (#4) = 1, is_true(t2.k (#0) = 12)] ├── estimated rows: 0.00 └── Window - ├── output columns: [test.k (#0), test.v (#1), rank() OVER (ORDER BY v DESC) (#4)] + ├── output columns: [test.k (#0), test.v (#1), rank() OVER ( ORDER BY v DESC ) (#4)] ├── aggregate function: [rank] ├── partition by: [] ├── order by: [v] @@ -212,11 +212,11 @@ query T explain select max(a) OVER (partition by a) FROM t qualify max(a) OVER (partition by a) > 3; ---- Filter -├── output columns: [max(a) OVER (PARTITION BY a) (#1)] -├── filters: [is_true(max(a) OVER (PARTITION BY a) (#1) > 3)] +├── output columns: [max(a) OVER ( PARTITION BY a ) (#1)] +├── filters: [is_true(max(a) OVER ( PARTITION BY a ) (#1) > 3)] ├── estimated rows: 0.00 └── Window - ├── output columns: [t.a (#0), max(a) OVER (PARTITION BY a) (#1)] + ├── output columns: [t.a (#0), max(a) OVER ( PARTITION BY a ) (#1)] ├── aggregate function: [max(a)] ├── partition by: [a] ├── order by: [] @@ -248,7 +248,7 @@ query T explain select b, row_number() over (order by b) from tbpush where b > 3; ---- Window -├── output columns: [tbpush.b (#0), row_number() OVER (ORDER BY b) (#1)] +├── output columns: [tbpush.b (#0), row_number() OVER ( ORDER BY b ) (#1)] ├── aggregate function: [row_number] ├── partition by: [] ├── order by: [b] @@ -294,11 +294,11 @@ query T explain select * from (select b, row_number() over (order by b) from tbpush) where b > 3; ---- Filter -├── output columns: [tbpush.b (#0), row_number() OVER (ORDER BY b) (#1)] +├── output columns: [tbpush.b (#0), row_number() OVER ( ORDER BY b ) (#1)] ├── filters: [is_true(tbpush.b (#0) > 3)] ├── estimated rows: 0.00 └── Window - ├── output columns: [tbpush.b (#0), row_number() OVER (ORDER BY b) (#1)] + ├── output columns: [tbpush.b (#0), row_number() OVER ( ORDER BY b ) (#1)] ├── aggregate function: [row_number] ├── partition by: [] ├── order by: [b] @@ -451,20 +451,20 @@ query T explain select *, sum(a) over (partition by a order by a desc rows between unbounded preceding and current row) from t where a > 1 order by b limit 3 ---- RowFetch -├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER (PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) (#6), t.c (#2), t.d (#3), t.e (#4), t.f (#5)] +├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER ( PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) (#6), t.c (#2), t.d (#3), t.e (#4), t.f (#5)] ├── columns to fetch: [c, d, e, f] ├── estimated rows: 0.00 └── Limit - ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER (PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) (#6)] + ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER ( PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) (#6)] ├── limit: 3 ├── offset: 0 ├── estimated rows: 0.00 └── Sort - ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER (PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) (#6)] + ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER ( PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) (#6)] ├── sort keys: [b ASC NULLS LAST] ├── estimated rows: 0.00 └── Window - ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER (PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) (#6)] + ├── output columns: [t.a (#0), t.b (#1), t._row_id (#7), sum(a) OVER ( PARTITION BY a ORDER BY a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) (#6)] ├── aggregate function: [sum(a)] ├── partition by: [a] ├── order by: [a] From 07a7d6d3995992237fa33cb61c623ab3397f8999 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Fri, 22 Mar 2024 21:41:24 +0800 Subject: [PATCH 08/10] fix --- .../mode/standalone/explain_native/explain.test | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test b/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test index a6f228fc79362..6bcb266d4694e 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test @@ -198,10 +198,10 @@ COPY INTO t1 FROM 's3://mybucket/data.csv' FILE_FORMAT = ( - field_delimiter = ",", - record_delimiter = "\n", - skip_header = "1", - type = "CSV" + field_delimiter = ',', + record_delimiter = '\n', + skip_header = 1, + type = CSV ) SIZE_LIMIT = 10 PURGE = false @@ -215,10 +215,10 @@ COPY INTO Uri(UriLocation { protocol: "s3", name: "mybucket", path: "/data.csv", part_prefix: "", connection: Connection { visited_keys: {}, conns: {} } }) FROM t1 FILE_FORMAT = ( - field_delimiter = ",", - record_delimiter = "\n", - skip_header = "1", - type = "CSV" + field_delimiter = ',', + record_delimiter = '\n', + skip_header = 1, + type = CSV ) SINGLE = false From 38a9c3b019a87f7c3d907e96d67f09fda49a7713 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Sat, 23 Mar 2024 01:44:05 +0800 Subject: [PATCH 09/10] fix --- Cargo.lock | 1 + src/query/ast/Cargo.toml | 1 + src/query/ast/src/ast/statements/copy.rs | 14 +++++++++- src/query/ast/tests/it/parser.rs | 1 + src/query/ast/tests/it/testdata/statement.txt | 28 +++++++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 341c121ed49d3..1d3f82406c268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2738,6 +2738,7 @@ dependencies = [ "pretty", "pretty_assertions", "regex", + "serde_json", "strsim 0.10.0", "strum 0.24.1", "strum_macros 0.24.3", diff --git a/src/query/ast/Cargo.toml b/src/query/ast/Cargo.toml index 524303bf0e94b..568367a3ca030 100644 --- a/src/query/ast/Cargo.toml +++ b/src/query/ast/Cargo.toml @@ -30,6 +30,7 @@ nom-rule = "0.3.0" ordered-float = { workspace = true } pratt = "0.4.0" pretty = "0.11.3" +serde_json = { workspace = true } strsim = "0.10" strum = "0.24" strum_macros = "0.24" diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index 57d7ca6eadbc8..0dfd359217330 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -505,7 +505,7 @@ impl FileFormatOptions { let options = self .options .iter() - .map(|(k, v)| (k.clone(), v.to_string())) + .map(|(k, v)| (k.clone(), v.to_meta_value())) .collect(); FileFormatOptionsAst { options } } @@ -526,6 +526,18 @@ pub enum FileFormatValue { StringList(Vec), } +impl FileFormatValue { + pub fn to_meta_value(&self) -> String { + match self { + FileFormatValue::Keyword(v) => v.clone(), + FileFormatValue::Bool(v) => v.to_string(), + FileFormatValue::U64(v) => v.to_string(), + FileFormatValue::String(v) => v.clone(), + FileFormatValue::StringList(v) => serde_json::to_string(v).unwrap(), + } + } +} + impl Display for FileFormatValue { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index abe872ef39f21..deb4fb69e212d 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -646,6 +646,7 @@ fn test_statement() { "CREATE OR REPLACE FUNCTION isnotempty_test_replace AS(p) -> not(is_null(p)) DESC = 'This is a description';", "CREATE FUNCTION binary_reverse (BINARY) RETURNS BINARY LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815';", "CREATE OR REPLACE FUNCTION binary_reverse (BINARY) RETURNS BINARY LANGUAGE python HANDLER = 'binary_reverse' ADDRESS = 'http://0.0.0.0:8815';", + r#"CREATE STAGE s file_format=(record_delimiter='\n' escape='\\');"#, r#"create or replace function addone(int) returns int language python diff --git a/src/query/ast/tests/it/testdata/statement.txt b/src/query/ast/tests/it/testdata/statement.txt index 30588e58d7968..0f0fdde4bfcc2 100644 --- a/src/query/ast/tests/it/testdata/statement.txt +++ b/src/query/ast/tests/it/testdata/statement.txt @@ -18378,6 +18378,34 @@ CreateUDF( ) +---------- Input ---------- +CREATE STAGE s file_format=(record_delimiter='\n' escape='\\'); +---------- Output --------- +CREATE STAGE s FILE_FORMAT = (escape = '\\', record_delimiter = '\n') +---------- AST ------------ +CreateStage( + CreateStageStmt { + create_option: Create, + stage_name: "s", + location: None, + file_format_options: FileFormatOptions { + options: { + "escape": String( + "\\", + ), + "record_delimiter": String( + "\n", + ), + }, + }, + on_error: "", + size_limit: 0, + validation_mode: "", + comments: "", + }, +) + + ---------- Input ---------- create or replace function addone(int) returns int From 1641841f9a800219e47db5ad33ce6f052a97d865 Mon Sep 17 00:00:00 2001 From: Andy Lok Date: Sat, 23 Mar 2024 13:28:04 +0800 Subject: [PATCH 10/10] fix --- src/query/ast/src/ast/statements/copy.rs | 6 +++++- src/query/sql/src/planner/binder/ddl/table.rs | 2 +- .../suites/mode/standalone/explain_native/explain.test | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/query/ast/src/ast/statements/copy.rs b/src/query/ast/src/ast/statements/copy.rs index 0dfd359217330..a8065a7e791f0 100644 --- a/src/query/ast/src/ast/statements/copy.rs +++ b/src/query/ast/src/ast/statements/copy.rs @@ -431,7 +431,11 @@ impl UriLocation { impl Display for UriLocation { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "'{}://{}{}'", self.protocol, self.name, self.path)?; - write!(f, "{}", self.connection)?; + if f.alternate() { + write!(f, "{}", self.connection.mask())?; + } else { + write!(f, "{}", self.connection)?; + } if !self.part_prefix.is_empty() { write!(f, " LOCATION_PREFIX = '{}'", self.part_prefix)?; } diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 7bd2ca7af8e77..610784512f156 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -707,7 +707,7 @@ impl Binder { // keep a copy of table data uri_location, will be used in "show create table" options.insert( OPT_KEY_TABLE_ATTACHED_DATA_URI.to_string(), - stmt.uri_location.to_string(), + format!("{:#}", stmt.uri_location), ); let mut uri = stmt.uri_location.clone(); diff --git a/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test b/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test index 6bcb266d4694e..447852b3ecf4a 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test +++ b/tests/sqllogictests/suites/mode/standalone/explain_native/explain.test @@ -455,10 +455,10 @@ CopyIntoTable (children 7) ├── FROM (children 1) │ └── Location 's3://mybucket/data.csv' ├── FileFormats (children 4) -│ ├── FileFormat field_delimiter = "," -│ ├── FileFormat record_delimiter = "\n" -│ ├── FileFormat skip_header = "1" -│ └── FileFormat type = "CSV" +│ ├── FileFormat field_delimiter = String(",") +│ ├── FileFormat record_delimiter = String("\n") +│ ├── FileFormat skip_header = U64(1) +│ └── FileFormat type = Keyword("CSV") ├── SizeLimit 10 ├── MaxFiles 10 ├── Purge false