feat(ast): add parser for template sql

src/query/ast/src/ast/common.rs

@@ -30,33 +30,39 @@ pub struct Identifier {
     pub name: String,
     #[drive(skip)]
     pub quote: Option<char>,
+    #[drive(skip)]
+    pub is_hole: bool,
 }
 
 impl Identifier {
     pub fn is_quoted(&self) -> bool {
         self.quote.is_some()
     }
 
-    pub fn from_name(name: impl Into<String>) -> Self {
+    pub fn from_name(span: Span, name: impl Into<String>) -> Self {
         Self {
-            span: Span::default(),
+            span,
             name: name.into(),
             quote: None,
+            is_hole: false,
         }
     }
 
-    pub fn from_name_with_quoted(name: impl Into<String>, quote: Option<char>) -> Self {
+    pub fn from_name_with_quoted(span: Span, name: impl Into<String>, quote: Option<char>) -> Self {
         Self {
-            span: Span::default(),
+            span,
             name: name.into(),
             quote,
+            is_hole: false,
         }
     }
 }
 
 impl Display for Identifier {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        if let Some(c) = self.quote {
+        if self.is_hole {
+            write!(f, "IDENTIFIER(:{})", self.name)
+        } else if let Some(c) = self.quote {
             let quoted = quote_ident(&self.name, c, true);
             write!(f, "{}", quoted)
         } else {

src/query/ast/src/ast/expr.rs

@@ -257,6 +257,12 @@ pub enum Expr {
         unit: IntervalKind,
         date: Box<Expr>,
     },
+    Hole {
+        #[drive(skip)]
+        span: Span,
+        #[drive(skip)]
+        name: String,
+    },
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Drive, DriveMut)]
@@ -625,7 +631,8 @@ impl Expr {
             | Expr::Interval { span, .. }
             | Expr::DateAdd { span, .. }
             | Expr::DateSub { span, .. }
-            | Expr::DateTrunc { span, .. } => *span,
+            | Expr::DateTrunc { span, .. }
+            | Expr::Hole { span, .. } => *span,
         }
     }
 
@@ -1358,6 +1365,9 @@ impl Display for Expr {
             Expr::DateTrunc { unit, date, .. } => {
                 write!(f, "DATE_TRUNC({unit}, {date})")?;
             }
+            Expr::Hole { name, .. } => {
+                write!(f, ":{name}")?;
+            }
         }
 
         Ok(())

src/query/ast/src/ast/format/syntax/expr.rs

@@ -417,5 +417,6 @@ pub(crate) fn pretty_expr(expr: Expr) -> RcDoc<'static> {
             .append(RcDoc::space())
             .append(pretty_expr(*date))
             .append(RcDoc::text(")")),
+        Expr::Hole { name, .. } => RcDoc::text(":").append(RcDoc::text(name.to_string())),
     }
 }

src/query/ast/src/ast/visitors/walk.rs

@@ -149,6 +149,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expr: &'a Expr) {
             unit,
         } => visitor.visit_date_sub(*span, unit, interval, date),
         Expr::DateTrunc { span, unit, date } => visitor.visit_date_trunc(*span, unit, date),
+        Expr::Hole { .. } => {}
     }
 }
 

src/query/ast/src/ast/visitors/walk_mut.rs

@@ -149,6 +149,7 @@ pub fn walk_expr_mut<V: VisitorMut>(visitor: &mut V, expr: &mut Expr) {
             unit,
         } => visitor.visit_date_sub(*span, unit, interval, date),
         Expr::DateTrunc { span, unit, date } => visitor.visit_date_trunc(*span, unit, date),
+        Expr::Hole { .. } => {}
     }
 }
 

src/query/ast/src/parser/common.rs

@@ -15,6 +15,7 @@
 use databend_common_exception::Range;
 use databend_common_exception::Span;
 use nom::branch::alt;
+use nom::combinator::consumed;
 use nom::combinator::map;
 use nom::multi::many1;
 use nom::sequence::terminated;
@@ -48,7 +49,7 @@ macro_rules! rule {
 }
 
 pub fn match_text(text: &'static str) -> impl FnMut(Input) -> IResult<&Token> {
-    move |i| match i.0.first().filter(|token| token.text() == text) {
+    move |i| match i.tokens.first().filter(|token| token.text() == text) {
         Some(token) => Ok((i.slice(1..), token)),
         _ => Err(nom::Err::Error(Error::from_error_kind(
             i,
@@ -58,7 +59,7 @@ pub fn match_text(text: &'static str) -> impl FnMut(Input) -> IResult<&Token> {
 }
 
 pub fn match_token(kind: TokenKind) -> impl FnMut(Input) -> IResult<&Token> {
-    move |i| match i.0.first().filter(|token| token.kind == kind) {
+    move |i| match i.tokens.first().filter(|token| token.kind == kind) {
         Some(token) => Ok((i.slice(1..), token)),
         _ => Err(nom::Err::Error(Error::from_error_kind(
             i,
@@ -68,7 +69,7 @@ pub fn match_token(kind: TokenKind) -> impl FnMut(Input) -> IResult<&Token> {
 }
 
 pub fn any_token(i: Input) -> IResult<&Token> {
-    match i.0.first().filter(|token| token.kind != EOI) {
+    match i.tokens.first().filter(|token| token.kind != EOI) {
         Some(token) => Ok((i.slice(1..), token)),
         _ => Err(nom::Err::Error(Error::from_error_kind(
             i,
@@ -102,10 +103,8 @@ pub fn function_name(i: Input) -> IResult<Identifier> {
 }
 
 pub fn stage_name(i: Input) -> IResult<Identifier> {
-    let anonymous_stage = map(rule! { "~" }, |token| Identifier {
-        span: transform_span(&[token.clone()]),
-        name: token.text().to_string(),
-        quote: None,
+    let anonymous_stage = map(consumed(rule! { "~" }), |(span, _)| {
+        Identifier::from_name(transform_span(span.tokens), "~")
     });
 
     rule!(
@@ -114,20 +113,40 @@ pub fn stage_name(i: Input) -> IResult<Identifier> {
     )(i)
 }
 
+fn plain_identifier(
+    is_reserved_keyword: fn(&TokenKind) -> bool,
+) -> impl FnMut(Input) -> IResult<Identifier> {
+    move |i| {
+        map(
+            rule! {
+                Ident
+                | #non_reserved_keyword(is_reserved_keyword)
+            },
+            |token| Identifier {
+                span: transform_span(&[token.clone()]),
+                name: token.text().to_string(),
+                quote: None,
+                is_hole: false,
+            },
+        )(i)
+    }
+}
+
 fn quoted_identifier(i: Input) -> IResult<Identifier> {
     match_token(QuotedString)(i).and_then(|(i2, token)| {
         if token
             .text()
             .chars()
             .next()
-            .filter(|c| i.1.is_ident_quote(*c))
+            .filter(|c| i.dialect.is_ident_quote(*c))
             .is_some()
         {
             let quote = token.text().chars().next().unwrap();
             Ok((i2, Identifier {
                 span: transform_span(&[token.clone()]),
                 name: unquote_ident(token.text(), quote),
                 quote: Some(quote),
+                is_hole: false,
             }))
         } else {
             Err(nom::Err::Error(Error::from_error_kind(
@@ -138,32 +157,37 @@ fn quoted_identifier(i: Input) -> IResult<Identifier> {
     })
 }
 
+fn identifier_hole(i: Input) -> IResult<Identifier> {
+    check_template_mode(map(
+        consumed(rule! {
+            IDENTIFIER ~ ^"(" ~ ^#template_hole ~ ^")"
+        }),
+        |(span, (_, _, (_, name), _))| Identifier {
+            span: transform_span(span.tokens),
+            name,
+            quote: None,
+            is_hole: true,
+        },
+    ))(i)
+}
+
 fn non_reserved_identifier(
     is_reserved_keyword: fn(&TokenKind) -> bool,
 ) -> impl FnMut(Input) -> IResult<Identifier> {
     move |i| {
-        alt((
-            map(
-                rule! {
-                    Ident
-                    | #non_reserved_keyword(is_reserved_keyword)
-                },
-                |token| Identifier {
-                    span: transform_span(&[token.clone()]),
-                    name: token.text().to_string(),
-                    quote: None,
-                },
-            ),
-            quoted_identifier,
-        ))(i)
+        rule!(
+            #plain_identifier(is_reserved_keyword)
+            | #quoted_identifier
+            | #identifier_hole
+        )(i)
     }
 }
 
 fn non_reserved_keyword(
     is_reserved_keyword: fn(&TokenKind) -> bool,
 ) -> impl FnMut(Input) -> IResult<&Token> {
     move |i: Input| match i
-        .0
+        .tokens
         .first()
         .filter(|token| token.kind.is_keyword() && !is_reserved_keyword(&token.kind))
     {
@@ -435,7 +459,7 @@ where
         .parse_input(&mut iter, Precedence(0))
         .map_err(|err| {
             // Rollback parsing footprint on unused expr elements.
-            input.2.clear();
+            input.backtrace.clear();
 
             let err_kind = match err {
                 PrattError::EmptyInput => ErrorKind::Other("expecting more subsequent tokens"),
@@ -462,13 +486,40 @@ where
         })?;
     if let Some(elem) = iter.peek() {
         // Rollback parsing footprint on unused expr elements.
-        input.2.clear();
+        input.backtrace.clear();
         Ok((input.slice(input.offset(&elem.span)..), expr))
     } else {
         Ok((rest, expr))
     }
 }
 
+pub fn check_template_mode<'a, O, F>(mut parser: F) -> impl FnMut(Input<'a>) -> IResult<'a, O>
+where F: nom::Parser<Input<'a>, O, Error<'a>> {
+    move |input: Input| {
+        parser.parse(input).and_then(|(i, res)| {
+            if input.mode.is_template() {
+                Ok((i, res))
+            } else {
+                i.backtrace.clear();
+                let error = Error::from_error_kind(
+                    input,
+                    ErrorKind::Other("variable is only available in SQL template"),
+                );
+                Err(nom::Err::Failure(error))
+            }
+        })
+    }
+}
+
+pub fn template_hole(i: Input) -> IResult<(Span, String)> {
+    check_template_mode(map(
+        consumed(rule! {
+            ":" ~ ^#plain_identifier(|token| token.is_reserved_ident(false))
+        }),
+        |(span, (_, ident))| (transform_span(span.tokens), ident.name),
+    ))(i)
+}
+
 macro_rules! declare_experimental_feature {
     ($check_fn_name: ident, $feature_name: literal) => {
         pub fn $check_fn_name<'a, O, F>(
@@ -480,10 +531,10 @@ macro_rules! declare_experimental_feature {
         {
             move |input: Input| {
                 parser.parse(input).and_then(|(i, res)| {
-                    if input.1.is_experimental() {
+                    if input.dialect.is_experimental() {
                         Ok((i, res))
                     } else {
-                        i.2.clear();
+                        i.backtrace.clear();
                         let error = Error::from_error_kind(
                             input,
                             ErrorKind::Other(

src/query/ast/src/parser/error.rs

@@ -89,7 +89,7 @@ impl<'a> nom::error::ParseError<Input<'a>> for Error<'a> {
             span: transform_span(&i[..1]).unwrap(),
             errors: vec![],
             contexts: vec![],
-            backtrace: i.2,
+            backtrace: i.backtrace,
         }
     }
 
@@ -118,39 +118,39 @@ impl<'a> nom::error::ContextError<Input<'a>> for Error<'a> {
     fn add_context(input: Input<'a>, ctx: &'static str, mut other: Self) -> Self {
         other
             .contexts
-            .push((transform_span(&input.0[..1]).unwrap(), ctx));
+            .push((transform_span(&input.tokens[..1]).unwrap(), ctx));
         other
     }
 }
 
 impl<'a> Error<'a> {
     pub fn from_error_kind(input: Input<'a>, kind: ErrorKind) -> Self {
-        let mut inner = input.2.inner.borrow_mut();
+        let mut inner = input.backtrace.inner.borrow_mut();
         if let Some(ref mut inner) = *inner {
-            match input.0[0].span.start.cmp(&inner.span.start) {
+            match input.tokens[0].span.start.cmp(&inner.span.start) {
                 Ordering::Equal => {
                     inner.errors.push(kind);
                 }
                 Ordering::Less => (),
                 Ordering::Greater => {
                     *inner = BacktraceInner {
-                        span: transform_span(&input.0[..1]).unwrap(),
+                        span: transform_span(&input.tokens[..1]).unwrap(),
                         errors: vec![kind],
                     };
                 }
             }
         } else {
             *inner = Some(BacktraceInner {
-                span: transform_span(&input.0[..1]).unwrap(),
+                span: transform_span(&input.tokens[..1]).unwrap(),
                 errors: vec![kind],
             })
         }
 
         Error {
-            span: transform_span(&input.0[..1]).unwrap(),
+            span: transform_span(&input.tokens[..1]).unwrap(),
             errors: vec![kind],
             contexts: vec![],
-            backtrace: input.2,
+            backtrace: input.backtrace,
         }
     }
 }