diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 0573240a2..47385b858 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -33,6 +33,8 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; +use crate::tokenizer::{Span, TokenWithLocation}; + pub use self::data_type::{ ArrayElemTypeDef, CharLengthUnits, CharacterLength, DataType, ExactNumberInfo, StructBracketKind, TimezoneInfo, @@ -86,6 +88,9 @@ mod dml; pub mod helpers; mod operator; mod query; +mod spans; +pub use spans::Spanned; + mod trigger; mod value; @@ -130,7 +135,7 @@ where } /// An identifier, decomposed into its value or character data and the quote style. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[derive(Debug, Clone, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Ident { @@ -139,8 +144,39 @@ pub struct Ident { /// The starting quote if any. Valid quote characters are the single quote, /// double quote, backtick, and opening square bracket. pub quote_style: Option, + /// The span of the identifier in the original SQL string. + pub span: Span, +} + +impl PartialEq for Ident { + fn eq(&self, other: &Self) -> bool { + let Ident { + value, + quote_style, + // backwards compat + span: _, + } = self; + + value == &other.value && quote_style == &other.quote_style + } +} + +impl core::hash::Hash for Ident { + fn hash(&self, state: &mut H) { + let Ident { + value, + quote_style, + // backwards compat + span: _, + } = self; + + value.hash(state); + quote_style.hash(state); + } } +impl Eq for Ident {} + impl Ident { /// Create a new identifier with the given value and no quotes. pub fn new(value: S) -> Self @@ -150,6 +186,7 @@ impl Ident { Ident { value: value.into(), quote_style: None, + span: Span::empty(), } } @@ -163,6 +200,30 @@ impl Ident { Ident { value: value.into(), quote_style: Some(quote), + span: Span::empty(), + } + } + + pub fn with_span(span: Span, value: S) -> Self + where + S: Into, + { + Ident { + value: value.into(), + quote_style: None, + span, + } + } + + pub fn with_quote_and_span(quote: char, span: Span, value: S) -> Self + where + S: Into, + { + assert!(quote == '\'' || quote == '"' || quote == '`' || quote == '['); + Ident { + value: value.into(), + quote_style: Some(quote), + span, } } } @@ -172,6 +233,7 @@ impl From<&str> for Ident { Ident { value: value.to_string(), quote_style: None, + span: Span::empty(), } } } @@ -901,10 +963,10 @@ pub enum Expr { /// `` opt_search_modifier: Option, }, - Wildcard, + Wildcard(TokenWithLocation), /// Qualified wildcard, e.g. `alias.*` or `schema.table.*`. /// (Same caveats apply to `QualifiedWildcard` as to `Wildcard`.) - QualifiedWildcard(ObjectName), + QualifiedWildcard(ObjectName, TokenWithLocation), /// Some dialects support an older syntax for outer joins where columns are /// marked with the `(+)` operator in the WHERE clause, for example: /// @@ -1193,8 +1255,8 @@ impl fmt::Display for Expr { Expr::MapAccess { column, keys } => { write!(f, "{column}{}", display_separated(keys, "")) } - Expr::Wildcard => f.write_str("*"), - Expr::QualifiedWildcard(prefix) => write!(f, "{}.*", prefix), + Expr::Wildcard(_) => f.write_str("*"), + Expr::QualifiedWildcard(prefix, _) => write!(f, "{}.*", prefix), Expr::CompoundIdentifier(s) => write!(f, "{}", display_separated(s, ".")), Expr::IsTrue(ast) => write!(f, "{ast} IS TRUE"), Expr::IsNotTrue(ast) => write!(f, "{ast} IS NOT TRUE"), @@ -5248,8 +5310,8 @@ pub enum FunctionArgExpr { impl From for FunctionArgExpr { fn from(wildcard_expr: Expr) -> Self { match wildcard_expr { - Expr::QualifiedWildcard(prefix) => Self::QualifiedWildcard(prefix), - Expr::Wildcard => Self::Wildcard, + Expr::QualifiedWildcard(prefix, _) => Self::QualifiedWildcard(prefix), + Expr::Wildcard(_) => Self::Wildcard, expr => Self::Expr(expr), } } diff --git a/src/ast/query.rs b/src/ast/query.rs index dc5966e5e..e8cc703cc 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -24,7 +24,10 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "visitor")] use sqlparser_derive::{Visit, VisitMut}; -use crate::ast::*; +use crate::{ + ast::*, + tokenizer::{Token, TokenWithLocation}, +}; /// The most complete variant of a `SELECT` query expression, optionally /// including `WITH`, `UNION` / other set operations, and `ORDER BY`. @@ -276,6 +279,8 @@ impl fmt::Display for Table { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct Select { + /// SELECT + pub select_token: TokenWithLocation, pub distinct: Option, /// MSSQL syntax: `TOP () [ PERCENT ] [ WITH TIES ]` pub top: Option, @@ -495,6 +500,7 @@ impl fmt::Display for NamedWindowDefinition { #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct With { + pub with_token: TokenWithLocation, pub recursive: bool, pub cte_tables: Vec, } @@ -546,6 +552,8 @@ pub struct Cte { pub query: Box, pub from: Option, pub materialized: Option, + // needed for accurate span reporting + pub closing_paren_token: TokenWithLocation, } impl fmt::Display for Cte { @@ -597,10 +605,11 @@ impl fmt::Display for IdentWithAlias { } /// Additional options for wildcards, e.g. Snowflake `EXCLUDE`/`RENAME` and Bigquery `EXCEPT`. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Default)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct WildcardAdditionalOptions { + pub wildcard_token: TokenWithLocation, /// `[ILIKE...]`. /// Snowflake syntax: pub opt_ilike: Option, @@ -618,6 +627,19 @@ pub struct WildcardAdditionalOptions { pub opt_rename: Option, } +impl Default for WildcardAdditionalOptions { + fn default() -> Self { + Self { + wildcard_token: TokenWithLocation::wrap(Token::Mul), + opt_ilike: None, + opt_exclude: None, + opt_except: None, + opt_replace: None, + opt_rename: None, + } + } +} + impl fmt::Display for WildcardAdditionalOptions { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if let Some(ilike) = &self.opt_ilike { diff --git a/src/ast/spans.rs b/src/ast/spans.rs new file mode 100644 index 000000000..3ded54ab2 --- /dev/null +++ b/src/ast/spans.rs @@ -0,0 +1,2136 @@ +use core::iter; + +use crate::tokenizer::Span; + +use super::{ + AlterColumnOperation, AlterIndexOperation, AlterTableOperation, Array, Assignment, + AssignmentTarget, CloseCursor, ClusteredIndex, ColumnDef, ColumnOption, ColumnOptionDef, + ConflictTarget, ConnectBy, ConstraintCharacteristics, CopySource, CreateIndex, CreateTable, + CreateTableOptions, Cte, Delete, DoUpdate, ExceptSelectItem, ExcludeSelectItem, Expr, + ExprWithAlias, Fetch, FromTable, Function, FunctionArg, FunctionArgExpr, + FunctionArgumentClause, FunctionArgumentList, FunctionArguments, GroupByExpr, HavingBound, + IlikeSelectItem, Insert, Interpolate, InterpolateExpr, Join, JoinConstraint, JoinOperator, + JsonPath, JsonPathElem, LateralView, MatchRecognizePattern, Measure, NamedWindowDefinition, + ObjectName, Offset, OnConflict, OnConflictAction, OnInsert, OrderBy, OrderByExpr, Partition, + PivotValueSource, ProjectionSelect, Query, ReferentialAction, RenameSelectItem, + ReplaceSelectElement, ReplaceSelectItem, Select, SelectInto, SelectItem, SetExpr, SqlOption, + Statement, Subscript, SymbolDefinition, TableAlias, TableConstraint, TableFactor, + TableOptionsClustered, TableWithJoins, Use, Value, Values, ViewColumnDef, + WildcardAdditionalOptions, With, WithFill, +}; + +/// Given an iterator of spans, return the [Span::union] of all spans. +fn union_spans>(iter: I) -> Span { + iter.reduce(|acc, item| acc.union(&item)) + .unwrap_or(Span::empty()) +} + +/// A trait for AST nodes that have a source span for use in diagnostics. +/// +/// Source spans are not guaranteed to be entirely accurate. They may +/// be missing keywords or other tokens. Some nodes may not have a computable +/// span at all, in which case they return [`Span::empty()`]. +/// +/// Some impl blocks may contain doc comments with information +/// on which nodes are missing spans. +pub trait Spanned { + /// Compute the source span for this AST node, by recursively + /// combining the spans of its children. + fn span(&self) -> Span; +} + +impl Spanned for Query { + fn span(&self) -> Span { + let Query { + with, + body, + order_by, + limit, + limit_by, + offset, + fetch, + locks: _, // todo + for_clause: _, // todo, mssql specific + settings: _, // todo, clickhouse specific + format_clause: _, // todo, clickhouse specific + } = self; + + union_spans( + with.iter() + .map(|i| i.span()) + .chain(core::iter::once(body.span())) + .chain(order_by.as_ref().map(|i| i.span())) + .chain(limit.as_ref().map(|i| i.span())) + .chain(limit_by.iter().map(|i| i.span())) + .chain(offset.as_ref().map(|i| i.span())) + .chain(fetch.as_ref().map(|i| i.span())), + ) + } +} + +impl Spanned for Offset { + fn span(&self) -> Span { + let Offset { + value, + rows: _, // enum + } = self; + + value.span() + } +} + +impl Spanned for Fetch { + fn span(&self) -> Span { + let Fetch { + with_ties: _, // bool + percent: _, // bool + quantity, + } = self; + + quantity.as_ref().map_or(Span::empty(), |i| i.span()) + } +} + +impl Spanned for With { + fn span(&self) -> Span { + let With { + with_token, + recursive: _, // bool + cte_tables, + } = self; + + union_spans( + core::iter::once(with_token.span).chain(cte_tables.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for Cte { + fn span(&self) -> Span { + let Cte { + alias, + query, + from, + materialized: _, // enum + closing_paren_token, + } = self; + + union_spans( + core::iter::once(alias.span()) + .chain(core::iter::once(query.span())) + .chain(from.iter().map(|item| item.span)) + .chain(core::iter::once(closing_paren_token.span)), + ) + } +} + +/// # partial span +/// +/// [SetExpr::Table] is not implemented. +impl Spanned for SetExpr { + fn span(&self) -> Span { + match self { + SetExpr::Select(select) => select.span(), + SetExpr::Query(query) => query.span(), + SetExpr::SetOperation { + op: _, + set_quantifier: _, + left, + right, + } => left.span().union(&right.span()), + SetExpr::Values(values) => values.span(), + SetExpr::Insert(statement) => statement.span(), + SetExpr::Table(_) => Span::empty(), + SetExpr::Update(statement) => statement.span(), + } + } +} + +impl Spanned for Values { + fn span(&self) -> Span { + let Values { + explicit_row: _, // bool, + rows, + } = self; + + union_spans( + rows.iter() + .map(|row| union_spans(row.iter().map(|expr| expr.span()))), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [Statement::CopyIntoSnowflake] +/// - [Statement::CreateSecret] +/// - [Statement::CreateRole] +/// - [Statement::AlterRole] +/// - [Statement::AttachDatabase] +/// - [Statement::AttachDuckDBDatabase] +/// - [Statement::DetachDuckDBDatabase] +/// - [Statement::Drop] +/// - [Statement::DropFunction] +/// - [Statement::DropProcedure] +/// - [Statement::DropSecret] +/// - [Statement::Declare] +/// - [Statement::CreateExtension] +/// - [Statement::Fetch] +/// - [Statement::Flush] +/// - [Statement::Discard] +/// - [Statement::SetRole] +/// - [Statement::SetVariable] +/// - [Statement::SetTimeZone] +/// - [Statement::SetNames] +/// - [Statement::SetNamesDefault] +/// - [Statement::ShowFunctions] +/// - [Statement::ShowVariable] +/// - [Statement::ShowStatus] +/// - [Statement::ShowVariables] +/// - [Statement::ShowCreate] +/// - [Statement::ShowColumns] +/// - [Statement::ShowTables] +/// - [Statement::ShowCollation] +/// - [Statement::StartTransaction] +/// - [Statement::SetTransaction] +/// - [Statement::Comment] +/// - [Statement::Commit] +/// - [Statement::Rollback] +/// - [Statement::CreateSchema] +/// - [Statement::CreateDatabase] +/// - [Statement::CreateFunction] +/// - [Statement::CreateTrigger] +/// - [Statement::DropTrigger] +/// - [Statement::CreateProcedure] +/// - [Statement::CreateMacro] +/// - [Statement::CreateStage] +/// - [Statement::Assert] +/// - [Statement::Grant] +/// - [Statement::Revoke] +/// - [Statement::Deallocate] +/// - [Statement::Execute] +/// - [Statement::Prepare] +/// - [Statement::Kill] +/// - [Statement::ExplainTable] +/// - [Statement::Explain] +/// - [Statement::Savepoint] +/// - [Statement::ReleaseSavepoint] +/// - [Statement::Merge] +/// - [Statement::Cache] +/// - [Statement::UNCache] +/// - [Statement::CreateSequence] +/// - [Statement::CreateType] +/// - [Statement::Pragma] +/// - [Statement::LockTables] +/// - [Statement::UnlockTables] +/// - [Statement::Unload] +/// - [Statement::OptimizeTable] +impl Spanned for Statement { + fn span(&self) -> Span { + match self { + Statement::Analyze { + table_name, + partitions, + for_columns: _, + columns, + cache_metadata: _, + noscan: _, + compute_statistics: _, + } => union_spans( + core::iter::once(table_name.span()) + .chain(partitions.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(columns.iter().map(|i| i.span)), + ), + Statement::Truncate { + table_names, + partitions, + table: _, + only: _, + identity: _, + cascade: _, + } => union_spans( + table_names + .iter() + .map(|i| i.name.span()) + .chain(partitions.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ), + Statement::Msck { + table_name, + repair: _, + partition_action: _, + } => table_name.span(), + Statement::Query(query) => query.span(), + Statement::Insert(insert) => insert.span(), + Statement::Install { extension_name } => extension_name.span, + Statement::Load { extension_name } => extension_name.span, + Statement::Directory { + overwrite: _, + local: _, + path: _, + file_format: _, + source, + } => source.span(), + Statement::Call(function) => function.span(), + Statement::Copy { + source, + to: _, + target: _, + options: _, + legacy_options: _, + values: _, + } => source.span(), + Statement::CopyIntoSnowflake { + into: _, + from_stage: _, + from_stage_alias: _, + stage_params: _, + from_transformations: _, + files: _, + pattern: _, + file_format: _, + copy_options: _, + validation_mode: _, + } => Span::empty(), + Statement::Close { cursor } => match cursor { + CloseCursor::All => Span::empty(), + CloseCursor::Specific { name } => name.span, + }, + Statement::Update { + table, + assignments, + from, + selection, + returning, + } => union_spans( + core::iter::once(table.span()) + .chain(assignments.iter().map(|i| i.span())) + .chain(from.iter().map(|i| i.span())) + .chain(selection.iter().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ), + Statement::Delete(delete) => delete.span(), + Statement::CreateView { + or_replace: _, + materialized: _, + name, + columns, + query, + options, + cluster_by, + comment: _, + with_no_schema_binding: _, + if_not_exists: _, + temporary: _, + to, + } => union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span())) + .chain(core::iter::once(query.span())) + .chain(core::iter::once(options.span())) + .chain(cluster_by.iter().map(|i| i.span)) + .chain(to.iter().map(|i| i.span())), + ), + Statement::CreateTable(create_table) => create_table.span(), + Statement::CreateVirtualTable { + name, + if_not_exists: _, + module_name, + module_args, + } => union_spans( + core::iter::once(name.span()) + .chain(core::iter::once(module_name.span)) + .chain(module_args.iter().map(|i| i.span)), + ), + Statement::CreateIndex(create_index) => create_index.span(), + Statement::CreateRole { .. } => Span::empty(), + Statement::CreateSecret { .. } => Span::empty(), + Statement::AlterTable { + name, + if_exists: _, + only: _, + operations, + location: _, + on_cluster, + } => union_spans( + core::iter::once(name.span()) + .chain(operations.iter().map(|i| i.span())) + .chain(on_cluster.iter().map(|i| i.span)), + ), + Statement::AlterIndex { name, operation } => name.span().union(&operation.span()), + Statement::AlterView { + name, + columns, + query, + with_options, + } => union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span)) + .chain(core::iter::once(query.span())) + .chain(with_options.iter().map(|i| i.span())), + ), + Statement::AlterRole { .. } => Span::empty(), + Statement::AttachDatabase { .. } => Span::empty(), + Statement::AttachDuckDBDatabase { .. } => Span::empty(), + Statement::DetachDuckDBDatabase { .. } => Span::empty(), + Statement::Drop { .. } => Span::empty(), + Statement::DropFunction { .. } => Span::empty(), + Statement::DropProcedure { .. } => Span::empty(), + Statement::DropSecret { .. } => Span::empty(), + Statement::Declare { .. } => Span::empty(), + Statement::CreateExtension { .. } => Span::empty(), + Statement::Fetch { .. } => Span::empty(), + Statement::Flush { .. } => Span::empty(), + Statement::Discard { .. } => Span::empty(), + Statement::SetRole { .. } => Span::empty(), + Statement::SetVariable { .. } => Span::empty(), + Statement::SetTimeZone { .. } => Span::empty(), + Statement::SetNames { .. } => Span::empty(), + Statement::SetNamesDefault {} => Span::empty(), + Statement::ShowFunctions { .. } => Span::empty(), + Statement::ShowVariable { .. } => Span::empty(), + Statement::ShowStatus { .. } => Span::empty(), + Statement::ShowVariables { .. } => Span::empty(), + Statement::ShowCreate { .. } => Span::empty(), + Statement::ShowColumns { .. } => Span::empty(), + Statement::ShowTables { .. } => Span::empty(), + Statement::ShowCollation { .. } => Span::empty(), + Statement::Use(u) => u.span(), + Statement::StartTransaction { .. } => Span::empty(), + Statement::SetTransaction { .. } => Span::empty(), + Statement::Comment { .. } => Span::empty(), + Statement::Commit { .. } => Span::empty(), + Statement::Rollback { .. } => Span::empty(), + Statement::CreateSchema { .. } => Span::empty(), + Statement::CreateDatabase { .. } => Span::empty(), + Statement::CreateFunction { .. } => Span::empty(), + Statement::CreateTrigger { .. } => Span::empty(), + Statement::DropTrigger { .. } => Span::empty(), + Statement::CreateProcedure { .. } => Span::empty(), + Statement::CreateMacro { .. } => Span::empty(), + Statement::CreateStage { .. } => Span::empty(), + Statement::Assert { .. } => Span::empty(), + Statement::Grant { .. } => Span::empty(), + Statement::Revoke { .. } => Span::empty(), + Statement::Deallocate { .. } => Span::empty(), + Statement::Execute { .. } => Span::empty(), + Statement::Prepare { .. } => Span::empty(), + Statement::Kill { .. } => Span::empty(), + Statement::ExplainTable { .. } => Span::empty(), + Statement::Explain { .. } => Span::empty(), + Statement::Savepoint { .. } => Span::empty(), + Statement::ReleaseSavepoint { .. } => Span::empty(), + Statement::Merge { .. } => Span::empty(), + Statement::Cache { .. } => Span::empty(), + Statement::UNCache { .. } => Span::empty(), + Statement::CreateSequence { .. } => Span::empty(), + Statement::CreateType { .. } => Span::empty(), + Statement::Pragma { .. } => Span::empty(), + Statement::LockTables { .. } => Span::empty(), + Statement::UnlockTables => Span::empty(), + Statement::Unload { .. } => Span::empty(), + Statement::OptimizeTable { .. } => Span::empty(), + } + } +} + +impl Spanned for Use { + fn span(&self) -> Span { + match self { + Use::Catalog(object_name) => object_name.span(), + Use::Schema(object_name) => object_name.span(), + Use::Database(object_name) => object_name.span(), + Use::Warehouse(object_name) => object_name.span(), + Use::Object(object_name) => object_name.span(), + Use::Default => Span::empty(), + } + } +} + +impl Spanned for CreateTable { + fn span(&self) -> Span { + let CreateTable { + or_replace: _, // bool + temporary: _, // bool + external: _, // bool + global: _, // bool + if_not_exists: _, // bool + transient: _, // bool + volatile: _, // bool + name, + columns, + constraints, + hive_distribution: _, // hive specific + hive_formats: _, // hive specific + table_properties, + with_options, + file_format: _, // enum + location: _, // string, no span + query, + without_rowid: _, // bool + like, + clone, + engine: _, // todo + comment: _, // todo, no span + auto_increment_offset: _, // u32, no span + default_charset: _, // string, no span + collation: _, // string, no span + on_commit: _, // enum + on_cluster: _, // todo, clickhouse specific + primary_key: _, // todo, clickhouse specific + order_by: _, // todo, clickhouse specific + partition_by: _, // todo, BigQuery specific + cluster_by: _, // todo, BigQuery specific + clustered_by: _, // todo, Hive specific + options: _, // todo, BigQuery specific + strict: _, // bool + copy_grants: _, // bool + enable_schema_evolution: _, // bool + change_tracking: _, // bool + data_retention_time_in_days: _, // u64, no span + max_data_extension_time_in_days: _, // u64, no span + default_ddl_collation: _, // string, no span + with_aggregation_policy: _, // todo, Snowflake specific + with_row_access_policy: _, // todo, Snowflake specific + with_tags: _, // todo, Snowflake specific + } = self; + + union_spans( + core::iter::once(name.span()) + .chain(columns.iter().map(|i| i.span())) + .chain(constraints.iter().map(|i| i.span())) + .chain(table_properties.iter().map(|i| i.span())) + .chain(with_options.iter().map(|i| i.span())) + .chain(query.iter().map(|i| i.span())) + .chain(like.iter().map(|i| i.span())) + .chain(clone.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for ColumnDef { + fn span(&self) -> Span { + let ColumnDef { + name, + data_type: _, // enum + collation, + options, + } = self; + + union_spans( + core::iter::once(name.span) + .chain(collation.iter().map(|i| i.span())) + .chain(options.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for ColumnOptionDef { + fn span(&self) -> Span { + let ColumnOptionDef { name, option } = self; + + option.span().union_opt(&name.as_ref().map(|i| i.span)) + } +} + +impl Spanned for TableConstraint { + fn span(&self) -> Span { + match self { + TableConstraint::Unique { + name, + index_name, + index_type_display: _, + index_type: _, + columns, + index_options: _, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(index_name.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::PrimaryKey { + name, + index_name, + index_type: _, + columns, + index_options: _, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(index_name.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::ForeignKey { + name, + columns, + foreign_table, + referred_columns, + on_delete, + on_update, + characteristics, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)) + .chain(core::iter::once(foreign_table.span())) + .chain(referred_columns.iter().map(|i| i.span)) + .chain(on_delete.iter().map(|i| i.span())) + .chain(on_update.iter().map(|i| i.span())) + .chain(characteristics.iter().map(|i| i.span())), + ), + TableConstraint::Check { name, expr } => { + expr.span().union_opt(&name.as_ref().map(|i| i.span)) + } + TableConstraint::Index { + display_as_key: _, + name, + index_type: _, + columns, + } => union_spans( + name.iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)), + ), + TableConstraint::FulltextOrSpatial { + fulltext: _, + index_type_display: _, + opt_index_name, + columns, + } => union_spans( + opt_index_name + .iter() + .map(|i| i.span) + .chain(columns.iter().map(|i| i.span)), + ), + } + } +} + +impl Spanned for CreateIndex { + fn span(&self) -> Span { + let CreateIndex { + name, + table_name, + using, + columns, + unique: _, // bool + concurrently: _, // bool + if_not_exists: _, // bool + include, + nulls_distinct: _, // bool + with, + predicate, + } = self; + + union_spans( + name.iter() + .map(|i| i.span()) + .chain(core::iter::once(table_name.span())) + .chain(using.iter().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span())) + .chain(include.iter().map(|i| i.span)) + .chain(with.iter().map(|i| i.span())) + .chain(predicate.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [ColumnOption::Null] +/// - [ColumnOption::NotNull] +/// - [ColumnOption::Comment] +/// - [ColumnOption::Unique]ยจ +/// - [ColumnOption::DialectSpecific] +/// - [ColumnOption::Generated] +impl Spanned for ColumnOption { + fn span(&self) -> Span { + match self { + ColumnOption::Null => Span::empty(), + ColumnOption::NotNull => Span::empty(), + ColumnOption::Default(expr) => expr.span(), + ColumnOption::Materialized(expr) => expr.span(), + ColumnOption::Ephemeral(expr) => expr.as_ref().map_or(Span::empty(), |e| e.span()), + ColumnOption::Alias(expr) => expr.span(), + ColumnOption::Unique { .. } => Span::empty(), + ColumnOption::ForeignKey { + foreign_table, + referred_columns, + on_delete, + on_update, + characteristics, + } => union_spans( + core::iter::once(foreign_table.span()) + .chain(referred_columns.iter().map(|i| i.span)) + .chain(on_delete.iter().map(|i| i.span())) + .chain(on_update.iter().map(|i| i.span())) + .chain(characteristics.iter().map(|i| i.span())), + ), + ColumnOption::Check(expr) => expr.span(), + ColumnOption::DialectSpecific(_) => Span::empty(), + ColumnOption::CharacterSet(object_name) => object_name.span(), + ColumnOption::Comment(_) => Span::empty(), + ColumnOption::OnUpdate(expr) => expr.span(), + ColumnOption::Generated { .. } => Span::empty(), + ColumnOption::Options(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +/// # missing span +impl Spanned for ReferentialAction { + fn span(&self) -> Span { + Span::empty() + } +} + +/// # missing span +impl Spanned for ConstraintCharacteristics { + fn span(&self) -> Span { + let ConstraintCharacteristics { + deferrable: _, // bool + initially: _, // enum + enforced: _, // bool + } = self; + + Span::empty() + } +} + +/// # partial span +/// +/// Missing spans: +/// - [AlterColumnOperation::SetNotNull] +/// - [AlterColumnOperation::DropNotNull] +/// - [AlterColumnOperation::DropDefault] +/// - [AlterColumnOperation::AddGenerated] +impl Spanned for AlterColumnOperation { + fn span(&self) -> Span { + match self { + AlterColumnOperation::SetNotNull => Span::empty(), + AlterColumnOperation::DropNotNull => Span::empty(), + AlterColumnOperation::SetDefault { value } => value.span(), + AlterColumnOperation::DropDefault => Span::empty(), + AlterColumnOperation::SetDataType { + data_type: _, + using, + } => using.as_ref().map_or(Span::empty(), |u| u.span()), + AlterColumnOperation::AddGenerated { .. } => Span::empty(), + } + } +} + +impl Spanned for CopySource { + fn span(&self) -> Span { + match self { + CopySource::Table { + table_name, + columns, + } => union_spans( + core::iter::once(table_name.span()).chain(columns.iter().map(|i| i.span)), + ), + CopySource::Query(query) => query.span(), + } + } +} + +impl Spanned for Delete { + fn span(&self) -> Span { + let Delete { + tables, + from, + using, + selection, + returning, + order_by, + limit, + } = self; + + union_spans( + tables + .iter() + .map(|i| i.span()) + .chain(core::iter::once(from.span())) + .chain( + using + .iter() + .map(|u| union_spans(u.iter().map(|i| i.span()))), + ) + .chain(selection.iter().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(order_by.iter().map(|i| i.span())) + .chain(limit.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for FromTable { + fn span(&self) -> Span { + match self { + FromTable::WithFromKeyword(vec) => union_spans(vec.iter().map(|i| i.span())), + FromTable::WithoutKeyword(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ViewColumnDef { + fn span(&self) -> Span { + let ViewColumnDef { + name, + data_type: _, // todo, DataType + options, + } = self; + + union_spans( + core::iter::once(name.span) + .chain(options.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ) + } +} + +impl Spanned for SqlOption { + fn span(&self) -> Span { + match self { + SqlOption::Clustered(table_options_clustered) => table_options_clustered.span(), + SqlOption::Ident(ident) => ident.span, + SqlOption::KeyValue { key, value } => key.span.union(&value.span()), + SqlOption::Partition { + column_name, + range_direction: _, + for_values, + } => union_spans( + core::iter::once(column_name.span).chain(for_values.iter().map(|i| i.span())), + ), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [TableOptionsClustered::ColumnstoreIndex] +impl Spanned for TableOptionsClustered { + fn span(&self) -> Span { + match self { + TableOptionsClustered::ColumnstoreIndex => Span::empty(), + TableOptionsClustered::ColumnstoreIndexOrder(vec) => { + union_spans(vec.iter().map(|i| i.span)) + } + TableOptionsClustered::Index(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ClusteredIndex { + fn span(&self) -> Span { + let ClusteredIndex { + name, + asc: _, // bool + } = self; + + name.span + } +} + +impl Spanned for CreateTableOptions { + fn span(&self) -> Span { + match self { + CreateTableOptions::None => Span::empty(), + CreateTableOptions::With(vec) => union_spans(vec.iter().map(|i| i.span())), + CreateTableOptions::Options(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [AlterTableOperation::OwnerTo] +impl Spanned for AlterTableOperation { + fn span(&self) -> Span { + match self { + AlterTableOperation::AddConstraint(table_constraint) => table_constraint.span(), + AlterTableOperation::AddColumn { + column_keyword: _, + if_not_exists: _, + column_def, + column_position: _, + } => column_def.span(), + AlterTableOperation::AddProjection { + if_not_exists: _, + name, + select, + } => name.span.union(&select.span()), + AlterTableOperation::DropProjection { if_exists: _, name } => name.span, + AlterTableOperation::MaterializeProjection { + if_exists: _, + name, + partition, + } => name.span.union_opt(&partition.as_ref().map(|i| i.span)), + AlterTableOperation::ClearProjection { + if_exists: _, + name, + partition, + } => name.span.union_opt(&partition.as_ref().map(|i| i.span)), + AlterTableOperation::DisableRowLevelSecurity => Span::empty(), + AlterTableOperation::DisableRule { name } => name.span, + AlterTableOperation::DisableTrigger { name } => name.span, + AlterTableOperation::DropConstraint { + if_exists: _, + name, + cascade: _, + } => name.span, + AlterTableOperation::DropColumn { + column_name, + if_exists: _, + cascade: _, + } => column_name.span, + AlterTableOperation::AttachPartition { partition } => partition.span(), + AlterTableOperation::DetachPartition { partition } => partition.span(), + AlterTableOperation::FreezePartition { + partition, + with_name, + } => partition + .span() + .union_opt(&with_name.as_ref().map(|n| n.span)), + AlterTableOperation::UnfreezePartition { + partition, + with_name, + } => partition + .span() + .union_opt(&with_name.as_ref().map(|n| n.span)), + AlterTableOperation::DropPrimaryKey => Span::empty(), + AlterTableOperation::EnableAlwaysRule { name } => name.span, + AlterTableOperation::EnableAlwaysTrigger { name } => name.span, + AlterTableOperation::EnableReplicaRule { name } => name.span, + AlterTableOperation::EnableReplicaTrigger { name } => name.span, + AlterTableOperation::EnableRowLevelSecurity => Span::empty(), + AlterTableOperation::EnableRule { name } => name.span, + AlterTableOperation::EnableTrigger { name } => name.span, + AlterTableOperation::RenamePartitions { + old_partitions, + new_partitions, + } => union_spans( + old_partitions + .iter() + .map(|i| i.span()) + .chain(new_partitions.iter().map(|i| i.span())), + ), + AlterTableOperation::AddPartitions { + if_not_exists: _, + new_partitions, + } => union_spans(new_partitions.iter().map(|i| i.span())), + AlterTableOperation::DropPartitions { + partitions, + if_exists: _, + } => union_spans(partitions.iter().map(|i| i.span())), + AlterTableOperation::RenameColumn { + old_column_name, + new_column_name, + } => old_column_name.span.union(&new_column_name.span), + AlterTableOperation::RenameTable { table_name } => table_name.span(), + AlterTableOperation::ChangeColumn { + old_name, + new_name, + data_type: _, + options, + column_position: _, + } => union_spans( + core::iter::once(old_name.span) + .chain(core::iter::once(new_name.span)) + .chain(options.iter().map(|i| i.span())), + ), + AlterTableOperation::ModifyColumn { + col_name, + data_type: _, + options, + column_position: _, + } => { + union_spans(core::iter::once(col_name.span).chain(options.iter().map(|i| i.span()))) + } + AlterTableOperation::RenameConstraint { old_name, new_name } => { + old_name.span.union(&new_name.span) + } + AlterTableOperation::AlterColumn { column_name, op } => { + column_name.span.union(&op.span()) + } + AlterTableOperation::SwapWith { table_name } => table_name.span(), + AlterTableOperation::SetTblProperties { table_properties } => { + union_spans(table_properties.iter().map(|i| i.span())) + } + AlterTableOperation::OwnerTo { .. } => Span::empty(), + } + } +} + +impl Spanned for Partition { + fn span(&self) -> Span { + match self { + Partition::Identifier(ident) => ident.span, + Partition::Expr(expr) => expr.span(), + Partition::Part(expr) => expr.span(), + Partition::Partitions(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +impl Spanned for ProjectionSelect { + fn span(&self) -> Span { + let ProjectionSelect { + projection, + order_by, + group_by, + } = self; + + union_spans( + projection + .iter() + .map(|i| i.span()) + .chain(order_by.iter().map(|i| i.span())) + .chain(group_by.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for OrderBy { + fn span(&self) -> Span { + let OrderBy { exprs, interpolate } = self; + + union_spans( + exprs + .iter() + .map(|i| i.span()) + .chain(interpolate.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [GroupByExpr::All] +impl Spanned for GroupByExpr { + fn span(&self) -> Span { + match self { + GroupByExpr::All(_) => Span::empty(), + GroupByExpr::Expressions(exprs, _modifiers) => { + union_spans(exprs.iter().map(|i| i.span())) + } + } + } +} + +impl Spanned for Interpolate { + fn span(&self) -> Span { + let Interpolate { exprs } = self; + + union_spans(exprs.iter().flat_map(|i| i.iter().map(|e| e.span()))) + } +} + +impl Spanned for InterpolateExpr { + fn span(&self) -> Span { + let InterpolateExpr { column, expr } = self; + + column.span.union_opt(&expr.as_ref().map(|e| e.span())) + } +} + +impl Spanned for AlterIndexOperation { + fn span(&self) -> Span { + match self { + AlterIndexOperation::RenameIndex { index_name } => index_name.span(), + } + } +} + +/// # partial span +/// +/// Missing spans:ever +/// - [Insert::insert_alias] +impl Spanned for Insert { + fn span(&self) -> Span { + let Insert { + or: _, // enum, sqlite specific + ignore: _, // bool + into: _, // bool + table_name, + table_alias, + columns, + overwrite: _, // bool + source, + partitioned, + after_columns, + table: _, // bool + on, + returning, + replace_into: _, // bool + priority: _, // todo, mysql specific + insert_alias: _, // todo, mysql specific + } = self; + + union_spans( + core::iter::once(table_name.span()) + .chain(table_alias.as_ref().map(|i| i.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(source.as_ref().map(|q| q.span())) + .chain(partitioned.iter().flat_map(|i| i.iter().map(|k| k.span()))) + .chain(after_columns.iter().map(|i| i.span)) + .chain(on.as_ref().map(|i| i.span())) + .chain(returning.iter().flat_map(|i| i.iter().map(|k| k.span()))), + ) + } +} + +impl Spanned for OnInsert { + fn span(&self) -> Span { + match self { + OnInsert::DuplicateKeyUpdate(vec) => union_spans(vec.iter().map(|i| i.span())), + OnInsert::OnConflict(on_conflict) => on_conflict.span(), + } + } +} + +impl Spanned for OnConflict { + fn span(&self) -> Span { + let OnConflict { + conflict_target, + action, + } = self; + + action + .span() + .union_opt(&conflict_target.as_ref().map(|i| i.span())) + } +} + +impl Spanned for ConflictTarget { + fn span(&self) -> Span { + match self { + ConflictTarget::Columns(vec) => union_spans(vec.iter().map(|i| i.span)), + ConflictTarget::OnConstraint(object_name) => object_name.span(), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [OnConflictAction::DoNothing] +impl Spanned for OnConflictAction { + fn span(&self) -> Span { + match self { + OnConflictAction::DoNothing => Span::empty(), + OnConflictAction::DoUpdate(do_update) => do_update.span(), + } + } +} + +impl Spanned for DoUpdate { + fn span(&self) -> Span { + let DoUpdate { + assignments, + selection, + } = self; + + union_spans( + assignments + .iter() + .map(|i| i.span()) + .chain(selection.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for Assignment { + fn span(&self) -> Span { + let Assignment { target, value } = self; + + target.span().union(&value.span()) + } +} + +impl Spanned for AssignmentTarget { + fn span(&self) -> Span { + match self { + AssignmentTarget::ColumnName(object_name) => object_name.span(), + AssignmentTarget::Tuple(vec) => union_spans(vec.iter().map(|i| i.span())), + } + } +} + +/// # partial span +/// +/// Most expressions are missing keywords in their spans. +/// f.e. `IS NULL ` reports as `::span`. +/// +/// Missing spans: +/// - [Expr::TypedString] +/// - [Expr::MatchAgainst] # MySQL specific +/// - [Expr::RLike] # MySQL specific +/// - [Expr::Struct] # BigQuery specific +/// - [Expr::Named] # BigQuery specific +/// - [Expr::Dictionary] # DuckDB specific +/// - [Expr::Map] # DuckDB specific +/// - [Expr::Lambda] +impl Spanned for Expr { + fn span(&self) -> Span { + match self { + Expr::Identifier(ident) => ident.span, + Expr::CompoundIdentifier(vec) => union_spans(vec.iter().map(|i| i.span)), + Expr::CompositeAccess { expr, key } => expr.span().union(&key.span), + Expr::IsFalse(expr) => expr.span(), + Expr::IsNotFalse(expr) => expr.span(), + Expr::IsTrue(expr) => expr.span(), + Expr::IsNotTrue(expr) => expr.span(), + Expr::IsNull(expr) => expr.span(), + Expr::IsNotNull(expr) => expr.span(), + Expr::IsUnknown(expr) => expr.span(), + Expr::IsNotUnknown(expr) => expr.span(), + Expr::IsDistinctFrom(lhs, rhs) => lhs.span().union(&rhs.span()), + Expr::IsNotDistinctFrom(lhs, rhs) => lhs.span().union(&rhs.span()), + Expr::InList { + expr, + list, + negated: _, + } => union_spans( + core::iter::once(expr.span()).chain(list.iter().map(|item| item.span())), + ), + Expr::InSubquery { + expr, + subquery, + negated: _, + } => expr.span().union(&subquery.span()), + Expr::InUnnest { + expr, + array_expr, + negated: _, + } => expr.span().union(&array_expr.span()), + Expr::Between { + expr, + negated: _, + low, + high, + } => expr.span().union(&low.span()).union(&high.span()), + + Expr::BinaryOp { left, op: _, right } => left.span().union(&right.span()), + Expr::Like { + negated: _, + expr, + pattern, + escape_char: _, + } => expr.span().union(&pattern.span()), + Expr::ILike { + negated: _, + expr, + pattern, + escape_char: _, + } => expr.span().union(&pattern.span()), + Expr::SimilarTo { + negated: _, + expr, + pattern, + escape_char: _, + } => expr.span().union(&pattern.span()), + Expr::Ceil { expr, field: _ } => expr.span(), + Expr::Floor { expr, field: _ } => expr.span(), + Expr::Position { expr, r#in } => expr.span().union(&r#in.span()), + Expr::Overlay { + expr, + overlay_what, + overlay_from, + overlay_for, + } => expr + .span() + .union(&overlay_what.span()) + .union(&overlay_from.span()) + .union_opt(&overlay_for.as_ref().map(|i| i.span())), + Expr::Collate { expr, collation } => expr + .span() + .union(&union_spans(collation.0.iter().map(|i| i.span))), + Expr::Nested(expr) => expr.span(), + Expr::Value(value) => value.span(), + Expr::TypedString { .. } => Span::empty(), + Expr::MapAccess { column, keys } => column + .span() + .union(&union_spans(keys.iter().map(|i| i.key.span()))), + Expr::Function(function) => function.span(), + Expr::GroupingSets(vec) => { + union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))) + } + Expr::Cube(vec) => union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))), + Expr::Rollup(vec) => union_spans(vec.iter().flat_map(|i| i.iter().map(|k| k.span()))), + Expr::Tuple(vec) => union_spans(vec.iter().map(|i| i.span())), + Expr::Array(array) => array.span(), + Expr::MatchAgainst { .. } => Span::empty(), + Expr::JsonAccess { value, path } => value.span().union(&path.span()), + Expr::RLike { .. } => Span::empty(), + Expr::AnyOp { + left, + compare_op: _, + right, + } => left.span().union(&right.span()), + Expr::AllOp { + left, + compare_op: _, + right, + } => left.span().union(&right.span()), + Expr::UnaryOp { op: _, expr } => expr.span(), + Expr::Convert { + expr, + data_type: _, + charset, + target_before_value: _, + styles, + } => union_spans( + core::iter::once(expr.span()) + .chain(charset.as_ref().map(|i| i.span())) + .chain(styles.iter().map(|i| i.span())), + ), + Expr::Cast { + kind: _, + expr, + data_type: _, + format: _, + } => expr.span(), + Expr::AtTimeZone { + timestamp, + time_zone, + } => timestamp.span().union(&time_zone.span()), + Expr::Extract { + field: _, + syntax: _, + expr, + } => expr.span(), + Expr::Substring { + expr, + substring_from, + substring_for, + special: _, + } => union_spans( + core::iter::once(expr.span()) + .chain(substring_from.as_ref().map(|i| i.span())) + .chain(substring_for.as_ref().map(|i| i.span())), + ), + Expr::Trim { + expr, + trim_where: _, + trim_what, + trim_characters, + } => union_spans( + core::iter::once(expr.span()) + .chain(trim_what.as_ref().map(|i| i.span())) + .chain( + trim_characters + .as_ref() + .map(|items| union_spans(items.iter().map(|i| i.span()))), + ), + ), + Expr::IntroducedString { value, .. } => value.span(), + Expr::Case { + operand, + conditions, + results, + else_result, + } => union_spans( + operand + .as_ref() + .map(|i| i.span()) + .into_iter() + .chain(conditions.iter().map(|i| i.span())) + .chain(results.iter().map(|i| i.span())) + .chain(else_result.as_ref().map(|i| i.span())), + ), + Expr::Exists { subquery, .. } => subquery.span(), + Expr::Subquery(query) => query.span(), + Expr::Struct { .. } => Span::empty(), + Expr::Named { .. } => Span::empty(), + Expr::Dictionary(_) => Span::empty(), + Expr::Map(_) => Span::empty(), + Expr::Subscript { expr, subscript } => expr.span().union(&subscript.span()), + Expr::Interval(interval) => interval.value.span(), + Expr::Wildcard(token) => token.span, + Expr::QualifiedWildcard(object_name, token) => union_spans( + object_name + .0 + .iter() + .map(|i| i.span) + .chain(iter::once(token.span)), + ), + Expr::OuterJoin(expr) => expr.span(), + Expr::Prior(expr) => expr.span(), + Expr::Lambda(_) => Span::empty(), + } + } +} + +impl Spanned for Subscript { + fn span(&self) -> Span { + match self { + Subscript::Index { index } => index.span(), + Subscript::Slice { + lower_bound, + upper_bound, + stride, + } => union_spans( + [ + lower_bound.as_ref().map(|i| i.span()), + upper_bound.as_ref().map(|i| i.span()), + stride.as_ref().map(|i| i.span()), + ] + .into_iter() + .flatten(), + ), + } + } +} + +impl Spanned for ObjectName { + fn span(&self) -> Span { + let ObjectName(segments) = self; + + union_spans(segments.iter().map(|i| i.span)) + } +} + +impl Spanned for Array { + fn span(&self) -> Span { + let Array { + elem, + named: _, // bool + } = self; + + union_spans(elem.iter().map(|i| i.span())) + } +} + +impl Spanned for Function { + fn span(&self) -> Span { + let Function { + name, + parameters, + args, + filter, + null_treatment: _, // enum + over: _, // todo + within_group, + } = self; + + union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(iter::once(args.span())) + .chain(iter::once(parameters.span())) + .chain(filter.iter().map(|i| i.span())) + .chain(within_group.iter().map(|i| i.span())), + ) + } +} + +/// # partial span +/// +/// The span of [FunctionArguments::None] is empty. +impl Spanned for FunctionArguments { + fn span(&self) -> Span { + match self { + FunctionArguments::None => Span::empty(), + FunctionArguments::Subquery(query) => query.span(), + FunctionArguments::List(list) => list.span(), + } + } +} + +impl Spanned for FunctionArgumentList { + fn span(&self) -> Span { + let FunctionArgumentList { + duplicate_treatment: _, // enum + args, + clauses, + } = self; + + union_spans( + // # todo: duplicate-treatment span + args.iter() + .map(|i| i.span()) + .chain(clauses.iter().map(|i| i.span())), + ) + } +} + +impl Spanned for FunctionArgumentClause { + fn span(&self) -> Span { + match self { + FunctionArgumentClause::IgnoreOrRespectNulls(_) => Span::empty(), + FunctionArgumentClause::OrderBy(vec) => union_spans(vec.iter().map(|i| i.expr.span())), + FunctionArgumentClause::Limit(expr) => expr.span(), + FunctionArgumentClause::OnOverflow(_) => Span::empty(), + FunctionArgumentClause::Having(HavingBound(_kind, expr)) => expr.span(), + FunctionArgumentClause::Separator(value) => value.span(), + } + } +} + +/// # partial span +/// +/// see Spanned impl for JsonPathElem for more information +impl Spanned for JsonPath { + fn span(&self) -> Span { + let JsonPath { path } = self; + + union_spans(path.iter().map(|i| i.span())) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JsonPathElem::Dot] +impl Spanned for JsonPathElem { + fn span(&self) -> Span { + match self { + JsonPathElem::Dot { .. } => Span::empty(), + JsonPathElem::Bracket { key } => key.span(), + } + } +} + +impl Spanned for SelectItem { + fn span(&self) -> Span { + match self { + SelectItem::UnnamedExpr(expr) => expr.span(), + SelectItem::ExprWithAlias { expr, alias } => expr.span().union(&alias.span), + SelectItem::QualifiedWildcard(object_name, wildcard_additional_options) => union_spans( + object_name + .0 + .iter() + .map(|i| i.span) + .chain(iter::once(wildcard_additional_options.span())), + ), + SelectItem::Wildcard(wildcard_additional_options) => wildcard_additional_options.span(), + } + } +} + +impl Spanned for WildcardAdditionalOptions { + fn span(&self) -> Span { + let WildcardAdditionalOptions { + wildcard_token, + opt_ilike, + opt_exclude, + opt_except, + opt_replace, + opt_rename, + } = self; + + union_spans( + core::iter::once(wildcard_token.span) + .chain(opt_ilike.as_ref().map(|i| i.span())) + .chain(opt_exclude.as_ref().map(|i| i.span())) + .chain(opt_rename.as_ref().map(|i| i.span())) + .chain(opt_replace.as_ref().map(|i| i.span())) + .chain(opt_except.as_ref().map(|i| i.span())), + ) + } +} + +/// # missing span +impl Spanned for IlikeSelectItem { + fn span(&self) -> Span { + Span::empty() + } +} + +impl Spanned for ExcludeSelectItem { + fn span(&self) -> Span { + match self { + ExcludeSelectItem::Single(ident) => ident.span, + ExcludeSelectItem::Multiple(vec) => union_spans(vec.iter().map(|i| i.span)), + } + } +} + +impl Spanned for RenameSelectItem { + fn span(&self) -> Span { + match self { + RenameSelectItem::Single(ident) => ident.ident.span.union(&ident.alias.span), + RenameSelectItem::Multiple(vec) => { + union_spans(vec.iter().map(|i| i.ident.span.union(&i.alias.span))) + } + } + } +} + +impl Spanned for ExceptSelectItem { + fn span(&self) -> Span { + let ExceptSelectItem { + first_element, + additional_elements, + } = self; + + union_spans( + iter::once(first_element.span).chain(additional_elements.iter().map(|i| i.span)), + ) + } +} + +impl Spanned for ReplaceSelectItem { + fn span(&self) -> Span { + let ReplaceSelectItem { items } = self; + + union_spans(items.iter().map(|i| i.span())) + } +} + +impl Spanned for ReplaceSelectElement { + fn span(&self) -> Span { + let ReplaceSelectElement { + expr, + column_name, + as_keyword: _, // bool + } = self; + + expr.span().union(&column_name.span) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [TableFactor::JsonTable] +impl Spanned for TableFactor { + fn span(&self) -> Span { + match self { + TableFactor::Table { + name, + alias, + args: _, + with_hints: _, + version: _, + with_ordinality: _, + partitions: _, + } => union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(alias.as_ref().map(|alias| { + union_spans( + iter::once(alias.name.span).chain(alias.columns.iter().map(|i| i.span)), + ) + })), + ), + TableFactor::Derived { + lateral: _, + subquery, + alias, + } => subquery + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::TableFunction { expr, alias } => expr + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::UNNEST { + alias, + with_offset: _, + with_offset_alias, + array_exprs, + with_ordinality: _, + } => union_spans( + alias + .iter() + .map(|i| i.span()) + .chain(array_exprs.iter().map(|i| i.span())) + .chain(with_offset_alias.as_ref().map(|i| i.span)), + ), + TableFactor::NestedJoin { + table_with_joins, + alias, + } => table_with_joins + .span() + .union_opt(&alias.as_ref().map(|alias| alias.span())), + TableFactor::Function { + lateral: _, + name, + args, + alias, + } => union_spans( + name.0 + .iter() + .map(|i| i.span) + .chain(args.iter().map(|i| i.span())) + .chain(alias.as_ref().map(|alias| alias.span())), + ), + TableFactor::JsonTable { .. } => Span::empty(), + TableFactor::Pivot { + table, + aggregate_functions, + value_column, + value_source, + default_on_null, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(aggregate_functions.iter().map(|i| i.span())) + .chain(value_column.iter().map(|i| i.span)) + .chain(core::iter::once(value_source.span())) + .chain(default_on_null.as_ref().map(|i| i.span())) + .chain(alias.as_ref().map(|i| i.span())), + ), + TableFactor::Unpivot { + table, + value, + name, + columns, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(core::iter::once(value.span)) + .chain(core::iter::once(name.span)) + .chain(columns.iter().map(|i| i.span)) + .chain(alias.as_ref().map(|alias| alias.span())), + ), + TableFactor::MatchRecognize { + table, + partition_by, + order_by, + measures, + rows_per_match: _, + after_match_skip: _, + pattern, + symbols, + alias, + } => union_spans( + core::iter::once(table.span()) + .chain(partition_by.iter().map(|i| i.span())) + .chain(order_by.iter().map(|i| i.span())) + .chain(measures.iter().map(|i| i.span())) + .chain(core::iter::once(pattern.span())) + .chain(symbols.iter().map(|i| i.span())) + .chain(alias.as_ref().map(|i| i.span())), + ), + } + } +} + +impl Spanned for PivotValueSource { + fn span(&self) -> Span { + match self { + PivotValueSource::List(vec) => union_spans(vec.iter().map(|i| i.span())), + PivotValueSource::Any(vec) => union_spans(vec.iter().map(|i| i.span())), + PivotValueSource::Subquery(query) => query.span(), + } + } +} + +impl Spanned for ExprWithAlias { + fn span(&self) -> Span { + let ExprWithAlias { expr, alias } = self; + + expr.span().union_opt(&alias.as_ref().map(|i| i.span)) + } +} + +/// # missing span +impl Spanned for MatchRecognizePattern { + fn span(&self) -> Span { + Span::empty() + } +} + +impl Spanned for SymbolDefinition { + fn span(&self) -> Span { + let SymbolDefinition { symbol, definition } = self; + + symbol.span.union(&definition.span()) + } +} + +impl Spanned for Measure { + fn span(&self) -> Span { + let Measure { expr, alias } = self; + + expr.span().union(&alias.span) + } +} + +impl Spanned for OrderByExpr { + fn span(&self) -> Span { + let OrderByExpr { + expr, + asc: _, // bool + nulls_first: _, // bool + with_fill, + } = self; + + expr.span().union_opt(&with_fill.as_ref().map(|f| f.span())) + } +} + +impl Spanned for WithFill { + fn span(&self) -> Span { + let WithFill { from, to, step } = self; + + union_spans( + from.iter() + .map(|f| f.span()) + .chain(to.iter().map(|t| t.span())) + .chain(step.iter().map(|s| s.span())), + ) + } +} + +impl Spanned for FunctionArg { + fn span(&self) -> Span { + match self { + FunctionArg::Named { + name, + arg, + operator: _, + } => name.span.union(&arg.span()), + FunctionArg::Unnamed(arg) => arg.span(), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [FunctionArgExpr::Wildcard] +impl Spanned for FunctionArgExpr { + fn span(&self) -> Span { + match self { + FunctionArgExpr::Expr(expr) => expr.span(), + FunctionArgExpr::QualifiedWildcard(object_name) => { + union_spans(object_name.0.iter().map(|i| i.span)) + } + FunctionArgExpr::Wildcard => Span::empty(), + } + } +} + +impl Spanned for TableAlias { + fn span(&self) -> Span { + let TableAlias { name, columns } = self; + + union_spans(iter::once(name.span).chain(columns.iter().map(|i| i.span))) + } +} + +/// # missing span +/// +/// The span of a `Value` is currently not implemented, as doing so +/// requires a breaking changes, which may be done in a future release. +impl Spanned for Value { + fn span(&self) -> Span { + Span::empty() // # todo: Value needs to store spans before this is possible + } +} + +impl Spanned for Join { + fn span(&self) -> Span { + let Join { + relation, + global: _, // bool + join_operator, + } = self; + + relation.span().union(&join_operator.span()) + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JoinOperator::CrossJoin] +/// - [JoinOperator::CrossApply] +/// - [JoinOperator::OuterApply] +impl Spanned for JoinOperator { + fn span(&self) -> Span { + match self { + JoinOperator::Inner(join_constraint) => join_constraint.span(), + JoinOperator::LeftOuter(join_constraint) => join_constraint.span(), + JoinOperator::RightOuter(join_constraint) => join_constraint.span(), + JoinOperator::FullOuter(join_constraint) => join_constraint.span(), + JoinOperator::CrossJoin => Span::empty(), + JoinOperator::LeftSemi(join_constraint) => join_constraint.span(), + JoinOperator::RightSemi(join_constraint) => join_constraint.span(), + JoinOperator::LeftAnti(join_constraint) => join_constraint.span(), + JoinOperator::RightAnti(join_constraint) => join_constraint.span(), + JoinOperator::CrossApply => Span::empty(), + JoinOperator::OuterApply => Span::empty(), + JoinOperator::AsOf { + match_condition, + constraint, + } => match_condition.span().union(&constraint.span()), + } + } +} + +/// # partial span +/// +/// Missing spans: +/// - [JoinConstraint::Natural] +/// - [JoinConstraint::None] +impl Spanned for JoinConstraint { + fn span(&self) -> Span { + match self { + JoinConstraint::On(expr) => expr.span(), + JoinConstraint::Using(vec) => union_spans(vec.iter().map(|i| i.span)), + JoinConstraint::Natural => Span::empty(), + JoinConstraint::None => Span::empty(), + } + } +} + +impl Spanned for TableWithJoins { + fn span(&self) -> Span { + let TableWithJoins { relation, joins } = self; + + union_spans(core::iter::once(relation.span()).chain(joins.iter().map(|item| item.span()))) + } +} + +impl Spanned for Select { + fn span(&self) -> Span { + let Select { + select_token, + distinct: _, // todo + top: _, // todo, mysql specific + projection, + into, + from, + lateral_views, + prewhere, + selection, + group_by, + cluster_by, + distribute_by, + sort_by, + having, + named_window, + qualify, + window_before_qualify: _, // bool + value_table_mode: _, // todo, BigQuery specific + connect_by, + } = self; + + union_spans( + core::iter::once(select_token.span) + .chain(projection.iter().map(|item| item.span())) + .chain(into.iter().map(|item| item.span())) + .chain(from.iter().map(|item| item.span())) + .chain(lateral_views.iter().map(|item| item.span())) + .chain(prewhere.iter().map(|item| item.span())) + .chain(selection.iter().map(|item| item.span())) + .chain(core::iter::once(group_by.span())) + .chain(cluster_by.iter().map(|item| item.span())) + .chain(distribute_by.iter().map(|item| item.span())) + .chain(sort_by.iter().map(|item| item.span())) + .chain(having.iter().map(|item| item.span())) + .chain(named_window.iter().map(|item| item.span())) + .chain(qualify.iter().map(|item| item.span())) + .chain(connect_by.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for ConnectBy { + fn span(&self) -> Span { + let ConnectBy { + condition, + relationships, + } = self; + + union_spans( + core::iter::once(condition.span()).chain(relationships.iter().map(|item| item.span())), + ) + } +} + +impl Spanned for NamedWindowDefinition { + fn span(&self) -> Span { + let NamedWindowDefinition( + ident, + _, // todo: NamedWindowExpr + ) = self; + + ident.span + } +} + +impl Spanned for LateralView { + fn span(&self) -> Span { + let LateralView { + lateral_view, + lateral_view_name, + lateral_col_alias, + outer: _, // bool + } = self; + + union_spans( + core::iter::once(lateral_view.span()) + .chain(core::iter::once(lateral_view_name.span())) + .chain(lateral_col_alias.iter().map(|i| i.span)), + ) + } +} + +impl Spanned for SelectInto { + fn span(&self) -> Span { + let SelectInto { + temporary: _, // bool + unlogged: _, // bool + table: _, // bool + name, + } = self; + + name.span() + } +} + +#[cfg(test)] +pub mod tests { + use crate::dialect::{Dialect, GenericDialect, SnowflakeDialect}; + use crate::parser::Parser; + use crate::tokenizer::Span; + + use super::*; + + struct SpanTest<'a>(Parser<'a>, &'a str); + + impl<'a> SpanTest<'a> { + fn new(dialect: &'a dyn Dialect, sql: &'a str) -> Self { + Self(Parser::new(dialect).try_with_sql(sql).unwrap(), sql) + } + + // get the subsection of the source string that corresponds to the span + // only works on single-line strings + fn get_source(&self, span: Span) -> &'a str { + // lines in spans are 1-indexed + &self.1[(span.start.column as usize - 1)..(span.end.column - 1) as usize] + } + } + + #[test] + fn test_join() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT id, name FROM users LEFT JOIN companies ON users.company_id = companies.id", + ); + + let query = test.0.parse_select().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT id, name FROM users LEFT JOIN companies ON users.company_id = companies.id" + ); + + let join_span = query.from[0].joins[0].span(); + + // 'LEFT JOIN' missing + assert_eq!( + test.get_source(join_span), + "companies ON users.company_id = companies.id" + ); + } + + #[test] + pub fn test_union() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT a FROM postgres.public.source UNION SELECT a FROM postgres.public.source", + ); + + let query = test.0.parse_query().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT a FROM postgres.public.source UNION SELECT a FROM postgres.public.source" + ); + } + + #[test] + pub fn test_subquery() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "SELECT a FROM (SELECT a FROM postgres.public.source) AS b", + ); + + let query = test.0.parse_select().unwrap(); + let select_span = query.span(); + + assert_eq!( + test.get_source(select_span), + "SELECT a FROM (SELECT a FROM postgres.public.source) AS b" + ); + + let subquery_span = query.from[0].span(); + + // left paren missing + assert_eq!( + test.get_source(subquery_span), + "SELECT a FROM postgres.public.source) AS b" + ); + } + + #[test] + pub fn test_cte() { + let dialect = &GenericDialect; + let mut test = SpanTest::new(dialect, "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner"); + + let query = test.0.parse_query().unwrap(); + + let select_span = query.span(); + + assert_eq!(test.get_source(select_span), "WITH cte_outer AS (SELECT a FROM postgres.public.source), cte_ignored AS (SELECT a FROM cte_outer), cte_inner AS (SELECT a FROM cte_outer) SELECT a FROM cte_inner"); + } + + #[test] + pub fn test_snowflake_lateral_flatten() { + let dialect = &SnowflakeDialect; + let mut test = SpanTest::new(dialect, "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED"); + + let query = test.0.parse_select().unwrap(); + + let select_span = query.span(); + + assert_eq!(test.get_source(select_span), "SELECT FLATTENED.VALUE:field::TEXT AS FIELD FROM SNOWFLAKE.SCHEMA.SOURCE AS S, LATERAL FLATTEN(INPUT => S.JSON_ARRAY) AS FLATTENED"); + } + + #[test] + pub fn test_wildcard_from_cte() { + let dialect = &GenericDialect; + let mut test = SpanTest::new( + dialect, + "WITH cte AS (SELECT a FROM postgres.public.source) SELECT cte.* FROM cte", + ); + + let query = test.0.parse_query().unwrap(); + let cte_span = query.clone().with.unwrap().cte_tables[0].span(); + let cte_query_span = query.clone().with.unwrap().cte_tables[0].query.span(); + let body_span = query.body.span(); + + // the WITH keyboard is part of the query + assert_eq!( + test.get_source(cte_span), + "cte AS (SELECT a FROM postgres.public.source)" + ); + assert_eq!( + test.get_source(cte_query_span), + "SELECT a FROM postgres.public.source" + ); + + assert_eq!(test.get_source(body_span), "SELECT cte.* FROM cte"); + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a9a5b1df4..433f9957b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -371,7 +371,7 @@ impl<'a> Parser<'a> { .into_iter() .map(|token| TokenWithLocation { token, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }) .collect(); self.with_tokens_with_locations(tokens_with_locations) @@ -564,7 +564,7 @@ impl<'a> Parser<'a> { let mut export = false; if !dialect_of!(self is MySqlDialect | GenericDialect) { - return parser_err!("Unsupported statement FLUSH", self.peek_token().location); + return parser_err!("Unsupported statement FLUSH", self.peek_token().span.start); } let location = if self.parse_keyword(Keyword::NO_WRITE_TO_BINLOG) { @@ -865,7 +865,7 @@ impl<'a> Parser<'a> { t @ (Token::Word(_) | Token::SingleQuotedString(_)) => { if self.peek_token().token == Token::Period { let mut id_parts: Vec = vec![match t { - Token::Word(w) => w.to_ident(), + Token::Word(w) => w.to_ident(next_token.span), Token::SingleQuotedString(s) => Ident::with_quote('\'', s), _ => unreachable!(), // We matched above }]; @@ -873,13 +873,16 @@ impl<'a> Parser<'a> { while self.consume_token(&Token::Period) { let next_token = self.next_token(); match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), + Token::Word(w) => id_parts.push(w.to_ident(next_token.span)), Token::SingleQuotedString(s) => { // SQLite has single-quoted identifiers id_parts.push(Ident::with_quote('\'', s)) } Token::Mul => { - return Ok(Expr::QualifiedWildcard(ObjectName(id_parts))); + return Ok(Expr::QualifiedWildcard( + ObjectName(id_parts), + next_token, + )); } _ => { return self @@ -890,7 +893,7 @@ impl<'a> Parser<'a> { } } Token::Mul => { - return Ok(Expr::Wildcard); + return Ok(Expr::Wildcard(next_token)); } _ => (), }; @@ -969,7 +972,7 @@ impl<'a> Parser<'a> { // Note also that naively `SELECT date` looks like a syntax error because the `date` type // name is not followed by a string literal, but in fact in PostgreSQL it is a valid // expression that should parse as the column name "date". - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let opt_expr = self.maybe_parse(|parser| { match parser.parse_data_type()? { DataType::Interval => parser.parse_interval(), @@ -1006,7 +1009,7 @@ impl<'a> Parser<'a> { if dialect_of!(self is PostgreSqlDialect | GenericDialect) => { Ok(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), + name: ObjectName(vec![w.to_ident(next_token.span)]), parameters: FunctionArguments::None, args: FunctionArguments::None, null_treatment: None, @@ -1020,7 +1023,7 @@ impl<'a> Parser<'a> { | Keyword::CURRENT_DATE | Keyword::LOCALTIME | Keyword::LOCALTIMESTAMP => { - self.parse_time_functions(ObjectName(vec![w.to_ident()])) + self.parse_time_functions(ObjectName(vec![w.to_ident(next_token.span)])) } Keyword::CASE => self.parse_case_expr(), Keyword::CONVERT => self.parse_convert_expr(false), @@ -1045,7 +1048,7 @@ impl<'a> Parser<'a> { Keyword::CEIL => self.parse_ceil_floor_expr(true), Keyword::FLOOR => self.parse_ceil_floor_expr(false), Keyword::POSITION if self.peek_token().token == Token::LParen => { - self.parse_position_expr(w.to_ident()) + self.parse_position_expr(w.to_ident(next_token.span)) } Keyword::SUBSTRING => self.parse_substring_expr(), Keyword::OVERLAY => self.parse_overlay_expr(), @@ -1064,7 +1067,7 @@ impl<'a> Parser<'a> { let query = self.parse_query()?; self.expect_token(&Token::RParen)?; Ok(Expr::Function(Function { - name: ObjectName(vec![w.to_ident()]), + name: ObjectName(vec![w.to_ident(next_token.span)]), parameters: FunctionArguments::None, args: FunctionArguments::Subquery(query), filter: None, @@ -1092,17 +1095,17 @@ impl<'a> Parser<'a> { // identifier, a function call, or a simple identifier: _ => match self.peek_token().token { Token::LParen | Token::Period => { - let mut id_parts: Vec = vec![w.to_ident()]; - let mut ends_with_wildcard = false; + let mut id_parts: Vec = vec![w.to_ident(next_token.span)]; + let mut ending_wildcard: Option = None; while self.consume_token(&Token::Period) { let next_token = self.next_token(); match next_token.token { - Token::Word(w) => id_parts.push(w.to_ident()), + Token::Word(w) => id_parts.push(w.to_ident(next_token.span)), Token::Mul => { // Postgres explicitly allows funcnm(tablenm.*) and the // function array_agg traverses this control flow if dialect_of!(self is PostgreSqlDialect) { - ends_with_wildcard = true; + ending_wildcard = Some(next_token); break; } else { return self @@ -1119,8 +1122,8 @@ impl<'a> Parser<'a> { } } - if ends_with_wildcard { - Ok(Expr::QualifiedWildcard(ObjectName(id_parts))) + if let Some(ending_wildcard_token) = ending_wildcard { + Ok(Expr::QualifiedWildcard(ObjectName(id_parts), ending_wildcard_token)) } else if self.consume_token(&Token::LParen) { if dialect_of!(self is SnowflakeDialect | MsSqlDialect) && self.consume_tokens(&[Token::Plus, Token::RParen]) @@ -1153,11 +1156,11 @@ impl<'a> Parser<'a> { Token::Arrow if self.dialect.supports_lambda_functions() => { self.expect_token(&Token::Arrow)?; return Ok(Expr::Lambda(LambdaFunction { - params: OneOrManyWithParens::One(w.to_ident()), + params: OneOrManyWithParens::One(w.to_ident(next_token.span)), body: Box::new(self.parse_expr()?), })); } - _ => Ok(Expr::Identifier(w.to_ident())), + _ => Ok(Expr::Identifier(w.to_ident(next_token.span))), }, }, // End of Token::Word // array `[1, 2, 3]` @@ -1244,11 +1247,11 @@ impl<'a> Parser<'a> { } else { let tok = self.next_token(); let key = match tok.token { - Token::Word(word) => word.to_ident(), + Token::Word(word) => word.to_ident(tok.span), _ => { return parser_err!( format!("Expected identifier, found: {tok}"), - tok.location + tok.span.start ) } }; @@ -2120,7 +2123,7 @@ impl<'a> Parser<'a> { } else if self.dialect.require_interval_qualifier() { return parser_err!( "INTERVAL requires a unit after the literal value", - self.peek_token().location + self.peek_token().span.start ); } else { None @@ -2211,7 +2214,10 @@ impl<'a> Parser<'a> { let (fields, trailing_bracket) = self.parse_struct_type_def(Self::parse_struct_field_def)?; if trailing_bracket.0 { - return parser_err!("unmatched > in STRUCT literal", self.peek_token().location); + return parser_err!( + "unmatched > in STRUCT literal", + self.peek_token().span.start + ); } self.expect_token(&Token::LParen)?; @@ -2241,7 +2247,7 @@ impl<'a> Parser<'a> { if typed_syntax { return parser_err!("Typed syntax does not allow AS", { self.prev_token(); - self.peek_token().location + self.peek_token().span.start }); } let field_name = self.parse_identifier(false)?; @@ -2294,7 +2300,7 @@ impl<'a> Parser<'a> { // we've matched all field types for the current struct. // e.g. this is invalid syntax `STRUCT>>, INT>(NULL)` if trailing_bracket.0 { - return parser_err!("unmatched > in STRUCT definition", start_token.location); + return parser_err!("unmatched > in STRUCT definition", start_token.span.start); } }; @@ -2663,7 +2669,7 @@ impl<'a> Parser<'a> { format!( "Expected one of [=, >, <, =>, =<, !=] as comparison operator, found: {op}" ), - tok.location + tok.span.start ); }; @@ -2789,7 +2795,7 @@ impl<'a> Parser<'a> { // Can only happen if `get_next_precedence` got out of sync with this function _ => parser_err!( format!("No infix parser for token {:?}", tok.token), - tok.location + tok.span.start ), } } else if Token::DoubleColon == tok { @@ -2821,7 +2827,7 @@ impl<'a> Parser<'a> { // Can only happen if `get_next_precedence` got out of sync with this function parser_err!( format!("No infix parser for token {:?}", tok.token), - tok.location + tok.span.start ) } } @@ -3124,14 +3130,14 @@ impl<'a> Parser<'a> { index += 1; if let Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) = token { continue; } break token.cloned().unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }); }) } @@ -3144,13 +3150,13 @@ impl<'a> Parser<'a> { match self.tokens.get(index - 1) { Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) => continue, non_whitespace => { if n == 0 { return non_whitespace.cloned().unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }); } n -= 1; @@ -3172,7 +3178,7 @@ impl<'a> Parser<'a> { .cloned() .unwrap_or(TokenWithLocation { token: Token::EOF, - location: Location { line: 0, column: 0 }, + span: Span::empty(), }) } @@ -3185,7 +3191,7 @@ impl<'a> Parser<'a> { match self.tokens.get(self.index - 1) { Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) => continue, token => { return token @@ -3211,7 +3217,7 @@ impl<'a> Parser<'a> { self.index -= 1; if let Some(TokenWithLocation { token: Token::Whitespace(_), - location: _, + span: _, }) = self.tokens.get(self.index) { continue; @@ -3224,7 +3230,7 @@ impl<'a> Parser<'a> { pub fn expected(&self, expected: &str, found: TokenWithLocation) -> Result { parser_err!( format!("Expected: {expected}, found: {found}"), - found.location + found.span.start ) } @@ -3232,11 +3238,21 @@ impl<'a> Parser<'a> { /// true. Otherwise, no tokens are consumed and returns false. #[must_use] pub fn parse_keyword(&mut self, expected: Keyword) -> bool { + self.parse_keyword_token(expected).is_some() + } + + #[must_use] + pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option { match self.peek_token().token { - Token::Word(w) if expected == w.keyword => { - self.next_token(); - true - } + Token::Word(w) if expected == w.keyword => Some(self.next_token()), + _ => None, + } + } + + #[must_use] + pub fn peek_keyword(&mut self, expected: Keyword) -> bool { + match self.peek_token().token { + Token::Word(w) if expected == w.keyword => true, _ => false, } } @@ -3318,9 +3334,9 @@ impl<'a> Parser<'a> { /// If the current token is the `expected` keyword, consume the token. /// Otherwise, return an error. - pub fn expect_keyword(&mut self, expected: Keyword) -> Result<(), ParserError> { - if self.parse_keyword(expected) { - Ok(()) + pub fn expect_keyword(&mut self, expected: Keyword) -> Result { + if let Some(token) = self.parse_keyword_token(expected) { + Ok(token) } else { self.expected(format!("{:?}", &expected).as_str(), self.peek_token()) } @@ -3362,9 +3378,9 @@ impl<'a> Parser<'a> { } /// Bail out if the current token is not an expected keyword, or consume it if it is - pub fn expect_token(&mut self, expected: &Token) -> Result<(), ParserError> { - if self.consume_token(expected) { - Ok(()) + pub fn expect_token(&mut self, expected: &Token) -> Result { + if self.peek_token() == *expected { + Ok(self.next_token()) } else { self.expected(&expected.to_string(), self.peek_token()) } @@ -3532,7 +3548,7 @@ impl<'a> Parser<'a> { /// Parse either `ALL`, `DISTINCT` or `DISTINCT ON (...)`. Returns [`None`] if `ALL` is parsed /// and results in a [`ParserError`] if both `ALL` and `DISTINCT` are found. pub fn parse_all_or_distinct(&mut self) -> Result, ParserError> { - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let all = self.parse_keyword(Keyword::ALL); let distinct = self.parse_keyword(Keyword::DISTINCT); if !distinct { @@ -4611,7 +4627,7 @@ impl<'a> Parser<'a> { let loc = self .tokens .get(self.index - 1) - .map_or(Location { line: 0, column: 0 }, |t| t.location); + .map_or(Location { line: 0, column: 0 }, |t| t.span.start); match keyword { Keyword::AUTHORIZATION => { if authorization_owner.is_some() { @@ -4921,7 +4937,7 @@ impl<'a> Parser<'a> { let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let names = self.parse_comma_separated(|p| p.parse_object_name(false))?; - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let cascade = self.parse_keyword(Keyword::CASCADE); let restrict = self.parse_keyword(Keyword::RESTRICT); let purge = self.parse_keyword(Keyword::PURGE); @@ -5806,7 +5822,7 @@ impl<'a> Parser<'a> { let _ = self.consume_token(&Token::Eq); let next_token = self.next_token(); match next_token.token { - Token::Number(s, _) => Some(Self::parse::(s, next_token.location)?), + Token::Number(s, _) => Some(Self::parse::(s, next_token.span.start)?), _ => self.expected("literal int", next_token)?, } } else { @@ -6590,7 +6606,7 @@ impl<'a> Parser<'a> { "FULLTEXT or SPATIAL option without constraint name", TokenWithLocation { token: Token::make_keyword(&name.to_string()), - location: next_token.location, + span: next_token.span, }, ); } @@ -7299,7 +7315,7 @@ impl<'a> Parser<'a> { Expr::Function(f) => Ok(Statement::Call(f)), other => parser_err!( format!("Expected a simple procedure call but found: {other}"), - self.peek_token().location + self.peek_token().span.start ), } } else { @@ -7503,7 +7519,7 @@ impl<'a> Parser<'a> { let loc = self .tokens .get(self.index - 1) - .map_or(Location { line: 0, column: 0 }, |t| t.location); + .map_or(Location { line: 0, column: 0 }, |t| t.span.start); return parser_err!(format!("Expect a char, found {s:?}"), loc); } Ok(s.chars().next().unwrap()) @@ -7549,7 +7565,7 @@ impl<'a> Parser<'a> { /// Parse a literal value (numbers, strings, date/time, booleans) pub fn parse_value(&mut self) -> Result { let next_token = self.next_token(); - let location = next_token.location; + let span = next_token.span; match next_token.token { Token::Word(w) => match w.keyword { Keyword::TRUE => Ok(Value::Boolean(true)), @@ -7562,7 +7578,7 @@ impl<'a> Parser<'a> { "A value?", TokenWithLocation { token: Token::Word(w), - location, + span, }, )?, }, @@ -7570,14 +7586,14 @@ impl<'a> Parser<'a> { "a concrete value", TokenWithLocation { token: Token::Word(w), - location, + span, }, ), }, // The call to n.parse() returns a bigdecimal when the // bigdecimal feature is enabled, and is otherwise a no-op // (i.e., it returns the input string). - Token::Number(n, l) => Ok(Value::Number(Self::parse(n, location)?, l)), + Token::Number(n, l) => Ok(Value::Number(Self::parse(n, span.start)?, l)), Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())), Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())), Token::TripleSingleQuotedString(ref s) => { @@ -7621,7 +7637,7 @@ impl<'a> Parser<'a> { // This because snowflake allows numbers as placeholders let next_token = self.next_token(); let ident = match next_token.token { - Token::Word(w) => Ok(w.to_ident()), + Token::Word(w) => Ok(w.to_ident(next_token.span)), Token::Number(w, false) => Ok(Ident::new(w)), _ => self.expected("placeholder", next_token), }?; @@ -7632,7 +7648,7 @@ impl<'a> Parser<'a> { "a value", TokenWithLocation { token: unexpected, - location, + span, }, ), } @@ -7672,7 +7688,7 @@ impl<'a> Parser<'a> { fn parse_introduced_string_value(&mut self) -> Result { let next_token = self.next_token(); - let location = next_token.location; + let span = next_token.span; match next_token.token { Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())), Token::DoubleQuotedString(ref s) => Ok(Value::DoubleQuotedString(s.to_string())), @@ -7681,7 +7697,7 @@ impl<'a> Parser<'a> { "a string value", TokenWithLocation { token: unexpected, - location, + span, }, ), } @@ -7691,7 +7707,7 @@ impl<'a> Parser<'a> { pub fn parse_literal_uint(&mut self) -> Result { let next_token = self.next_token(); match next_token.token { - Token::Number(s, _) => Self::parse::(s, next_token.location), + Token::Number(s, _) => Self::parse::(s, next_token.span.start), _ => self.expected("literal int", next_token), } } @@ -8090,7 +8106,7 @@ impl<'a> Parser<'a> { // (For example, in `FROM t1 JOIN` the `JOIN` will always be parsed as a keyword, // not an alias.) Token::Word(w) if after_as || !reserved_kwds.contains(&w.keyword) => { - Ok(Some(w.to_ident())) + Ok(Some(w.to_ident(next_token.span))) } // MSSQL supports single-quoted strings as aliases for columns // We accept them as table aliases too, although MSSQL does not. @@ -8160,7 +8176,7 @@ impl<'a> Parser<'a> { _ => { return parser_err!( "BUG: expected to match GroupBy modifier keyword", - self.peek_token().location + self.peek_token().span.start ) } }); @@ -8223,6 +8239,7 @@ impl<'a> Parser<'a> { .map(|value| Ident { value: value.into(), quote_style: ident.quote_style, + span: ident.span, }) .collect::>() }) @@ -8238,7 +8255,7 @@ impl<'a> Parser<'a> { loop { match self.peek_token().token { Token::Word(w) => { - idents.push(w.to_ident()); + idents.push(w.to_ident(self.peek_token().span)); } Token::EOF | Token::Eq => break, _ => {} @@ -8291,8 +8308,9 @@ impl<'a> Parser<'a> { let mut idents = vec![]; // expecting at least one word for identifier - match self.next_token().token { - Token::Word(w) => idents.push(w.to_ident()), + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => idents.push(w.to_ident(next_token.span)), Token::EOF => { return Err(ParserError::ParserError( "Empty input when parsing identifier".to_string(), @@ -8309,19 +8327,22 @@ impl<'a> Parser<'a> { loop { match self.next_token().token { // ensure that optional period is succeeded by another identifier - Token::Period => match self.next_token().token { - Token::Word(w) => idents.push(w.to_ident()), - Token::EOF => { - return Err(ParserError::ParserError( - "Trailing period in identifier".to_string(), - ))? - } - token => { - return Err(ParserError::ParserError(format!( - "Unexpected token following period in identifier: {token}" - )))? + Token::Period => { + let next_token = self.next_token(); + match next_token.token { + Token::Word(w) => idents.push(w.to_ident(next_token.span)), + Token::EOF => { + return Err(ParserError::ParserError( + "Trailing period in identifier".to_string(), + ))? + } + token => { + return Err(ParserError::ParserError(format!( + "Unexpected token following period in identifier: {token}" + )))? + } } - }, + } Token::EOF => break, token => { return Err(ParserError::ParserError(format!( @@ -8343,7 +8364,7 @@ impl<'a> Parser<'a> { let next_token = self.next_token(); match next_token.token { Token::Word(w) => { - let mut ident = w.to_ident(); + let mut ident = w.to_ident(next_token.span); // On BigQuery, hyphens are permitted in unquoted identifiers inside of a FROM or // TABLE clause [0]. @@ -8759,8 +8780,9 @@ impl<'a> Parser<'a> { /// expect the initial keyword to be already consumed pub fn parse_query(&mut self) -> Result, ParserError> { let _guard = self.recursion_counter.try_decrease()?; - let with = if self.parse_keyword(Keyword::WITH) { + let with = if let Some(with_token) = self.parse_keyword_token(Keyword::WITH) { Some(With { + with_token, recursive: self.parse_keyword(Keyword::RECURSIVE), cte_tables: self.parse_comma_separated(Parser::parse_cte)?, }) @@ -9018,8 +9040,10 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; - self.expect_token(&Token::RParen)?; + let closing_paren_token = self.expect_token(&Token::RParen)?; + let alias = TableAlias { name, columns: vec![], @@ -9029,6 +9053,7 @@ impl<'a> Parser<'a> { query, from: None, materialized: is_materialized, + closing_paren_token, } } else { let columns = self.parse_parenthesized_column_list(Optional, false)?; @@ -9042,14 +9067,17 @@ impl<'a> Parser<'a> { } } self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; - self.expect_token(&Token::RParen)?; + let closing_paren_token = self.expect_token(&Token::RParen)?; + let alias = TableAlias { name, columns }; Cte { alias, query, from: None, materialized: is_materialized, + closing_paren_token, } }; if self.parse_keyword(Keyword::FROM) { @@ -9069,7 +9097,7 @@ impl<'a> Parser<'a> { pub fn parse_query_body(&mut self, precedence: u8) -> Result, ParserError> { // We parse the expression using a Pratt parser, as in `parse_expr()`. // Start by parsing a restricted SELECT or a `(subquery)`: - let expr = if self.parse_keyword(Keyword::SELECT) { + let expr = if self.peek_keyword(Keyword::SELECT) { SetExpr::Select(self.parse_select().map(Box::new)?) } else if self.consume_token(&Token::LParen) { // CTEs are not allowed here, but the parser currently accepts them @@ -9159,8 +9187,9 @@ impl<'a> Parser<'a> { } /// Parse a restricted `SELECT` statement (no CTEs / `UNION` / `ORDER BY`), - /// assuming the initial `SELECT` was already consumed + /// assuming the initial `SELECT` was not yet consumed pub fn parse_select(&mut self) -> Result { + let select_token = self.expect_keyword(Keyword::SELECT)?; let value_table_mode = if dialect_of!(self is BigQueryDialect) && self.parse_keyword(Keyword::AS) { if self.parse_keyword(Keyword::VALUE) { @@ -9321,6 +9350,7 @@ impl<'a> Parser<'a> { }; Ok(Select { + select_token, distinct, top, projection, @@ -10320,7 +10350,7 @@ impl<'a> Parser<'a> { return self.expected("literal number", next_token); }; self.expect_token(&Token::RBrace)?; - RepetitionQuantifier::AtMost(Self::parse(n, token.location)?) + RepetitionQuantifier::AtMost(Self::parse(n, token.span.start)?) } Token::Number(n, _) if self.consume_token(&Token::Comma) => { let next_token = self.next_token(); @@ -10328,12 +10358,12 @@ impl<'a> Parser<'a> { Token::Number(m, _) => { self.expect_token(&Token::RBrace)?; RepetitionQuantifier::Range( - Self::parse(n, token.location)?, - Self::parse(m, token.location)?, + Self::parse(n, token.span.start)?, + Self::parse(m, token.span.start)?, ) } Token::RBrace => { - RepetitionQuantifier::AtLeast(Self::parse(n, token.location)?) + RepetitionQuantifier::AtLeast(Self::parse(n, token.span.start)?) } _ => { return self.expected("} or upper bound", next_token); @@ -10342,7 +10372,7 @@ impl<'a> Parser<'a> { } Token::Number(n, _) => { self.expect_token(&Token::RBrace)?; - RepetitionQuantifier::Exactly(Self::parse(n, token.location)?) + RepetitionQuantifier::Exactly(Self::parse(n, token.span.start)?) } _ => return self.expected("quantifier range", token), } @@ -10733,7 +10763,7 @@ impl<'a> Parser<'a> { .parse_keywords(&[Keyword::GRANTED, Keyword::BY]) .then(|| self.parse_identifier(false).unwrap()); - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; let cascade = self.parse_keyword(Keyword::CASCADE); let restrict = self.parse_keyword(Keyword::RESTRICT); if cascade && restrict { @@ -10752,7 +10782,10 @@ impl<'a> Parser<'a> { /// Parse an REPLACE statement pub fn parse_replace(&mut self) -> Result { if !dialect_of!(self is MySqlDialect | GenericDialect) { - return parser_err!("Unsupported statement REPLACE", self.peek_token().location); + return parser_err!( + "Unsupported statement REPLACE", + self.peek_token().span.start + ); } let mut insert = self.parse_insert()?; @@ -11161,7 +11194,7 @@ impl<'a> Parser<'a> { } fn parse_duplicate_treatment(&mut self) -> Result, ParserError> { - let loc = self.peek_token().location; + let loc = self.peek_token().span.start; match ( self.parse_keyword(Keyword::ALL), self.parse_keyword(Keyword::DISTINCT), @@ -11176,17 +11209,17 @@ impl<'a> Parser<'a> { /// Parse a comma-delimited list of projections after SELECT pub fn parse_select_item(&mut self) -> Result { match self.parse_wildcard_expr()? { - Expr::QualifiedWildcard(prefix) => Ok(SelectItem::QualifiedWildcard( + Expr::QualifiedWildcard(prefix, token) => Ok(SelectItem::QualifiedWildcard( prefix, - self.parse_wildcard_additional_options()?, + self.parse_wildcard_additional_options(token)?, )), - Expr::Wildcard => Ok(SelectItem::Wildcard( - self.parse_wildcard_additional_options()?, + Expr::Wildcard(token) => Ok(SelectItem::Wildcard( + self.parse_wildcard_additional_options(token)?, )), Expr::Identifier(v) if v.value.to_lowercase() == "from" && v.quote_style.is_none() => { parser_err!( format!("Expected an expression, found: {}", v), - self.peek_token().location + self.peek_token().span.start ) } Expr::BinaryOp { @@ -11221,6 +11254,7 @@ impl<'a> Parser<'a> { /// If it is not possible to parse it, will return an option. pub fn parse_wildcard_additional_options( &mut self, + wildcard_token: TokenWithLocation, ) -> Result { let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) { self.parse_optional_select_item_ilike()? @@ -11252,6 +11286,7 @@ impl<'a> Parser<'a> { }; Ok(WildcardAdditionalOptions { + wildcard_token, opt_ilike, opt_exclude, opt_except, @@ -11499,7 +11534,7 @@ impl<'a> Parser<'a> { } else { let next_token = self.next_token(); let quantity = match next_token.token { - Token::Number(s, _) => Self::parse::(s, next_token.location)?, + Token::Number(s, _) => Self::parse::(s, next_token.span.start)?, _ => self.expected("literal int", next_token)?, }; Some(TopQuantity::Constant(quantity)) @@ -12227,10 +12262,11 @@ impl<'a> Parser<'a> { } impl Word { - pub fn to_ident(&self) -> Ident { + pub fn to_ident(&self, span: Span) -> Ident { Ident { value: self.value.clone(), quote_style: self.quote_style, + span, } } } @@ -12804,14 +12840,17 @@ mod tests { Ident { value: "CATALOG".to_string(), quote_style: None, + span: Span::empty(), }, Ident { value: "F(o)o. \"bar".to_string(), quote_style: Some('"'), + span: Span::empty(), }, Ident { value: "table".to_string(), quote_style: None, + span: Span::empty(), }, ]; dialect.run_parser_method(r#"CATALOG."F(o)o. ""bar".table"#, |parser| { @@ -12824,10 +12863,12 @@ mod tests { Ident { value: "CATALOG".to_string(), quote_style: None, + span: Span::empty(), }, Ident { value: "table".to_string(), quote_style: None, + span: Span::empty(), }, ]; dialect.run_parser_method("CATALOG . table", |parser| { diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 4186ec824..7fe48d824 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -29,10 +29,10 @@ use alloc::{ vec, vec::Vec, }; -use core::fmt; use core::iter::Peekable; use core::num::NonZeroU8; use core::str::Chars; +use core::{cmp, fmt}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -422,7 +422,8 @@ impl fmt::Display for Whitespace { } /// Location in input string -#[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[derive(Eq, PartialEq, Hash, Clone, Copy, Ord, PartialOrd)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Location { /// Line number, starting from 1 pub line: u64, @@ -430,37 +431,142 @@ pub struct Location { pub column: u64, } -impl fmt::Display for Location { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl std::fmt::Display for Location { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if self.line == 0 { return Ok(()); } - write!( - f, - // TODO: use standard compiler location syntax (::) - " at Line: {}, Column: {}", - self.line, self.column, - ) + write!(f, " at Line: {}, Column: {}", self.line, self.column) + } +} + +impl std::fmt::Debug for Location { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Location({},{})", self.line, self.column) + } +} + +impl Location { + pub fn of(line: u64, column: u64) -> Self { + Self { line, column } + } + + pub fn span_to(self, end: Self) -> Span { + Span { start: self, end } + } +} + +impl From<(u64, u64)> for Location { + fn from((line, column): (u64, u64)) -> Self { + Self { line, column } + } +} + +/// A span of source code locations (start, end) +#[derive(Eq, PartialEq, Hash, Clone, PartialOrd, Ord, Copy)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Span { + pub start: Location, + pub end: Location, +} + +impl std::fmt::Debug for Span { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Span({:?}..{:?})", self.start, self.end) + } +} + +impl Span { + // An empty span (0, 0) -> (0, 0) + // We need a const instance for pattern matching + const EMPTY: Span = Self::empty(); + + pub fn new(start: Location, end: Location) -> Span { + Span { start, end } + } + + /// Returns an empty span (0, 0) -> (0, 0) + /// Empty spans represent no knowledge of source location + pub const fn empty() -> Span { + Span { + start: Location { line: 0, column: 0 }, + end: Location { line: 0, column: 0 }, + } + } + + /// Returns the smallest Span that contains both `self` and `other` + /// If either span is [Span::empty], the other span is returned + pub fn union(&self, other: &Span) -> Span { + // If either span is empty, return the other + // this prevents propagating (0, 0) through the tree + match (self, other) { + (&Span::EMPTY, _) => *other, + (_, &Span::EMPTY) => *self, + _ => Span { + start: cmp::min(self.start, other.start), + end: cmp::max(self.end, other.end), + }, + } + } + + /// Same as [Span::union] for `Option` + /// If `other` is `None`, `self` is returned + pub fn union_opt(&self, other: &Option) -> Span { + match other { + Some(other) => self.union(other), + None => *self, + } } } /// A [Token] with [Location] attached to it -#[derive(Debug, Eq, PartialEq, Clone)] +#[derive(Debug, Eq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct TokenWithLocation { pub token: Token, - pub location: Location, + pub span: Span, } impl TokenWithLocation { - pub fn new(token: Token, line: u64, column: u64) -> TokenWithLocation { - TokenWithLocation { - token, - location: Location { line, column }, - } + pub fn new(token: Token, span: Span) -> TokenWithLocation { + TokenWithLocation { token, span } } pub fn wrap(token: Token) -> TokenWithLocation { - TokenWithLocation::new(token, 0, 0) + TokenWithLocation::new(token, Span::empty()) + } + + pub fn at(token: Token, start: Location, end: Location) -> TokenWithLocation { + TokenWithLocation::new(token, Span::new(start, end)) + } +} + +impl core::hash::Hash for TokenWithLocation { + fn hash(&self, state: &mut H) { + let TokenWithLocation { token, span: _ } = self; + + token.hash(state); + } +} + +impl PartialEq for TokenWithLocation { + fn eq(&self, other: &TokenWithLocation) -> bool { + let TokenWithLocation { token, span: _ } = self; + + token == &other.token + } +} + +impl PartialOrd for TokenWithLocation { + fn partial_cmp(&self, other: &TokenWithLocation) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for TokenWithLocation { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let TokenWithLocation { token, span: _ } = self; + token.cmp(&other.token) } } @@ -656,7 +762,9 @@ impl<'a> Tokenizer<'a> { let mut location = state.location(); while let Some(token) = self.next_token(&mut state)? { - buf.push(TokenWithLocation { token, location }); + let span = location.span_to(state.location()); + + buf.push(TokenWithLocation { token, span }); location = state.location(); } @@ -2668,13 +2776,25 @@ mod tests { .tokenize_with_location() .unwrap(); let expected = vec![ - TokenWithLocation::new(Token::make_keyword("SELECT"), 1, 1), - TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 1, 7), - TokenWithLocation::new(Token::make_word("a", None), 1, 8), - TokenWithLocation::new(Token::Comma, 1, 9), - TokenWithLocation::new(Token::Whitespace(Whitespace::Newline), 1, 10), - TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 2, 1), - TokenWithLocation::new(Token::make_word("b", None), 2, 2), + TokenWithLocation::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Space), + (1, 7).into(), + (1, 8).into(), + ), + TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()), + TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Newline), + (1, 10).into(), + (2, 1).into(), + ), + TokenWithLocation::at( + Token::Whitespace(Whitespace::Space), + (2, 1).into(), + (2, 2).into(), + ), + TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()), ]; compare(expected, tokens); } diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 2bf470f71..91d49de54 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -23,6 +23,7 @@ use std::ops::Deref; use sqlparser::ast::*; use sqlparser::dialect::{BigQueryDialect, GenericDialect}; use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use test_utils::*; #[test] @@ -677,10 +678,12 @@ fn parse_typed_struct_syntax_bigquery() { Ident { value: "t".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "str_col".into(), quote_style: None, + span: Span::empty(), }, ]), ], @@ -689,6 +692,7 @@ fn parse_typed_struct_syntax_bigquery() { field_name: Some(Ident { value: "x".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::Int64 }, @@ -696,6 +700,7 @@ fn parse_typed_struct_syntax_bigquery() { field_name: Some(Ident { value: "y".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::String(None) }, @@ -708,6 +713,7 @@ fn parse_typed_struct_syntax_bigquery() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -739,6 +745,7 @@ fn parse_typed_struct_syntax_bigquery() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -986,10 +993,12 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { Ident { value: "t".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "str_col".into(), quote_style: None, + span: Span::empty(), }, ]), ], @@ -998,6 +1007,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { field_name: Some(Ident { value: "x".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::Int64 }, @@ -1005,6 +1015,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { field_name: Some(Ident { value: "y".into(), quote_style: None, + span: Span::empty(), }), field_type: DataType::String(None) }, @@ -1017,6 +1028,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { @@ -1048,6 +1060,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() { values: vec![Expr::Identifier(Ident { value: "nested_col".into(), quote_style: None, + span: Span::empty(), }),], fields: vec![ StructField { diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs index f8c349a37..bb238e1d3 100644 --- a/tests/sqlparser_clickhouse.rs +++ b/tests/sqlparser_clickhouse.rs @@ -21,6 +21,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::{Span, Token, TokenWithLocation}; use test_utils::*; use sqlparser::ast::Expr::{BinaryOp, Identifier, MapAccess}; @@ -39,11 +40,13 @@ fn parse_map_access_expr() { assert_eq!( Select { distinct: None, + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), top: None, projection: vec![UnnamedExpr(MapAccess { column: Box::new(Identifier(Ident { value: "string_values".to_string(), quote_style: None, + span: Span::empty(), })), keys: vec![MapAccessKey { key: call( @@ -900,7 +903,8 @@ fn parse_create_view_with_fields_data_types() { data_type: Some(DataType::Custom( ObjectName(vec![Ident { value: "int".into(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }]), vec![] )), @@ -911,7 +915,8 @@ fn parse_create_view_with_fields_data_types() { data_type: Some(DataType::Custom( ObjectName(vec![Ident { value: "String".into(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }]), vec![] )), diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index a2eb5070d..3397a08be 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -37,6 +37,7 @@ use sqlparser::dialect::{ use sqlparser::keywords::ALL_KEYWORDS; use sqlparser::parser::{Parser, ParserError, ParserOptions}; use sqlparser::tokenizer::Tokenizer; +use sqlparser::tokenizer::{Span, Token, TokenWithLocation}; use test_utils::{ all_dialects, all_dialects_where, alter_table_op, assert_eq_vec, call, expr_from_projection, join, number, only, table, table_alias, TestedDialects, @@ -377,6 +378,7 @@ fn parse_update_set_from() { subquery: Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![ @@ -1231,6 +1233,7 @@ fn parse_select_with_date_column_name() { &Expr::Identifier(Ident { value: "date".into(), quote_style: None, + span: Span::empty(), }), expr_from_projection(only(&select.projection)), ); @@ -1578,6 +1581,7 @@ fn parse_null_like() { alias: Ident { value: "col_null".to_owned(), quote_style: None, + span: Span::empty(), }, }, select.projection[0] @@ -1594,6 +1598,7 @@ fn parse_null_like() { alias: Ident { value: "null_col".to_owned(), quote_style: None, + span: Span::empty(), }, }, select.projection[1] @@ -2650,6 +2655,7 @@ fn parse_listagg() { expr: Expr::Identifier(Ident { value: "id".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -2659,6 +2665,7 @@ fn parse_listagg() { expr: Expr::Identifier(Ident { value: "username".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -3865,7 +3872,8 @@ fn parse_alter_table() { [SqlOption::KeyValue { key: Ident { value: "classification".to_string(), - quote_style: Some('\'') + quote_style: Some('\''), + span: Span::empty(), }, value: Expr::Value(Value::SingleQuotedString("parquet".to_string())), }], @@ -4647,6 +4655,7 @@ fn test_parse_named_window() { ORDER BY C3"; let actual_select_only = verified_only_select(sql); let expected = Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECTI")), distinct: None, top: None, projection: vec![ @@ -4655,6 +4664,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "MIN".to_string(), quote_style: None, + span: Span::empty(), }]), parameters: FunctionArguments::None, args: FunctionArguments::List(FunctionArgumentList { @@ -4663,6 +4673,7 @@ fn test_parse_named_window() { Expr::Identifier(Ident { value: "c12".to_string(), quote_style: None, + span: Span::empty(), }), ))], clauses: vec![], @@ -4672,12 +4683,14 @@ fn test_parse_named_window() { over: Some(WindowType::NamedWindow(Ident { value: "window1".to_string(), quote_style: None, + span: Span::empty(), })), within_group: vec![], }), alias: Ident { value: "min1".to_string(), quote_style: None, + span: Span::empty(), }, }, SelectItem::ExprWithAlias { @@ -4685,6 +4698,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "MAX".to_string(), quote_style: None, + span: Span::empty(), }]), parameters: FunctionArguments::None, args: FunctionArguments::List(FunctionArgumentList { @@ -4693,6 +4707,7 @@ fn test_parse_named_window() { Expr::Identifier(Ident { value: "c12".to_string(), quote_style: None, + span: Span::empty(), }), ))], clauses: vec![], @@ -4702,12 +4717,14 @@ fn test_parse_named_window() { over: Some(WindowType::NamedWindow(Ident { value: "window2".to_string(), quote_style: None, + span: Span::empty(), })), within_group: vec![], }), alias: Ident { value: "max1".to_string(), quote_style: None, + span: Span::empty(), }, }, ], @@ -4717,6 +4734,7 @@ fn test_parse_named_window() { name: ObjectName(vec![Ident { value: "aggregate_test_100".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -4740,6 +4758,7 @@ fn test_parse_named_window() { Ident { value: "window1".to_string(), quote_style: None, + span: Span::empty(), }, NamedWindowExpr::WindowSpec(WindowSpec { window_name: None, @@ -4748,6 +4767,7 @@ fn test_parse_named_window() { expr: Expr::Identifier(Ident { value: "C12".to_string(), quote_style: None, + span: Span::empty(), }), asc: None, nulls_first: None, @@ -4760,12 +4780,14 @@ fn test_parse_named_window() { Ident { value: "window2".to_string(), quote_style: None, + span: Span::empty(), }, NamedWindowExpr::WindowSpec(WindowSpec { window_name: None, partition_by: vec![Expr::Identifier(Ident { value: "C11".to_string(), quote_style: None, + span: Span::empty(), })], order_by: vec![], window_frame: None, @@ -5247,6 +5269,7 @@ fn interval_disallow_interval_expr_gt() { right: Box::new(Expr::Identifier(Ident { value: "x".to_string(), quote_style: None, + span: Span::empty(), })), } ) @@ -5287,11 +5310,13 @@ fn parse_interval_and_or_xor() { let expected_ast = vec![Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![UnnamedExpr(Expr::Identifier(Ident { value: "col".to_string(), quote_style: None, + span: Span::empty(), }))], into: None, from: vec![TableWithJoins { @@ -5299,6 +5324,7 @@ fn parse_interval_and_or_xor() { name: ObjectName(vec![Ident { value: "test".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -5316,12 +5342,14 @@ fn parse_interval_and_or_xor() { left: Box::new(Expr::Identifier(Ident { value: "d3_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "d1_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Plus, right: Box::new(Expr::Interval(Interval { @@ -5340,12 +5368,14 @@ fn parse_interval_and_or_xor() { left: Box::new(Expr::Identifier(Ident { value: "d2_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "d1_date".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Plus, right: Box::new(Expr::Interval(Interval { @@ -5437,6 +5467,7 @@ fn parse_at_timezone() { alias: Ident { value: "hour".to_string(), quote_style: Some('"'), + span: Span::empty(), }, }, only(&select.projection), @@ -6394,15 +6425,18 @@ fn parse_recursive_cte() { name: Ident { value: "nums".to_string(), quote_style: None, + span: Span::empty(), }, columns: vec![Ident { value: "val".to_string(), quote_style: None, + span: Span::empty(), }], }, query: Box::new(cte_query), from: None, materialized: None, + closing_paren_token: TokenWithLocation::wrap(Token::RParen), }; assert_eq!(with.cte_tables.first().unwrap(), &expected); } @@ -7365,21 +7399,17 @@ fn lateral_function() { let sql = "SELECT * FROM customer LEFT JOIN LATERAL generate_series(1, customer.id)"; let actual_select_only = verified_only_select(sql); let expected = Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "customer".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -8017,10 +8047,12 @@ fn parse_grant() { Ident { value: "shape".into(), quote_style: None, + span: Span::empty(), }, Ident { value: "size".into(), quote_style: None, + span: Span::empty(), }, ]) }, @@ -8213,6 +8245,7 @@ fn parse_merge() { subquery: Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![SelectItem::Wildcard( @@ -8259,6 +8292,7 @@ fn parse_merge() { name: Ident { value: "stg".to_string(), quote_style: None, + span: Span::empty(), }, columns: vec![], }), @@ -8458,7 +8492,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8472,7 +8507,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8486,7 +8522,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8496,7 +8533,8 @@ fn test_lock_table() { lock.of.unwrap().0, vec![Ident { value: "student".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert!(lock.nonblock.is_none()); @@ -8513,7 +8551,8 @@ fn test_lock_nonblock() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert_eq!(lock.nonblock.unwrap(), NonBlock::SkipLocked); @@ -8527,7 +8566,8 @@ fn test_lock_nonblock() { lock.of.unwrap().0, vec![Ident { value: "school".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }] ); assert_eq!(lock.nonblock.unwrap(), NonBlock::Nowait); @@ -9327,7 +9367,8 @@ fn parse_pivot_table() { alias: Some(TableAlias { name: Ident { value: "p".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, columns: vec![Ident::new("c"), Ident::new("d")], }), @@ -9375,12 +9416,14 @@ fn parse_unpivot_table() { }), value: Ident { value: "quantity".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, name: Ident { value: "quarter".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, columns: ["Q1", "Q2", "Q3", "Q4"] .into_iter() @@ -9442,12 +9485,14 @@ fn parse_pivot_unpivot_table() { }), value: Ident { value: "population".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, name: Ident { value: "year".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() }, columns: ["population_2000", "population_2010"] .into_iter() @@ -9801,6 +9846,7 @@ fn parse_unload() { Statement::Unload { query: Box::new(Query { body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![UnnamedExpr(Expr::Identifier(Ident::new("cola"))),], @@ -9844,12 +9890,14 @@ fn parse_unload() { }), to: Ident { value: "s3://...".to_string(), - quote_style: Some('\'') + quote_style: Some('\''), + span: Span::empty(), }, with: vec![SqlOption::KeyValue { key: Ident { value: "format".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, value: Expr::Value(Value::SingleQuotedString("AVRO".to_string())) }] @@ -9976,6 +10024,7 @@ fn parse_map_access_expr() { #[test] fn parse_connect_by() { let expect_query = Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![ @@ -10062,6 +10111,7 @@ fn parse_connect_by() { assert_eq!( all_dialects_where(|d| d.supports_connect_by()).verified_only_select(connect_by_3), Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![ @@ -10902,6 +10952,7 @@ fn test_extract_seconds_ok() { field: DateTimeField::Custom(Ident { value: "seconds".to_string(), quote_style: None, + span: Span::empty(), }), syntax: ExtractSyntax::From, expr: Box::new(Expr::Cast { @@ -10927,6 +10978,7 @@ fn test_extract_seconds_single_quote_ok() { field: DateTimeField::Custom(Ident { value: "seconds".to_string(), quote_style: Some('\''), + span: Span::empty(), }), syntax: ExtractSyntax::From, expr: Box::new(Expr::Cast { diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index a4109b0a3..d2874a8a2 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -18,6 +18,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::{Span, Token, TokenWithLocation}; use test_utils::*; use sqlparser::ast::*; @@ -259,21 +260,17 @@ fn test_select_union_by_name() { op: SetOperator::Union, set_quantifier: *expected_quantifier, left: Box::::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "capitals".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -299,21 +296,17 @@ fn test_select_union_by_name() { connect_by: None, }))), right: Box::::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, - projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions { - opt_ilike: None, - opt_exclude: None, - opt_except: None, - opt_rename: None, - opt_replace: None, - })], + projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())], into: None, from: vec![TableWithJoins { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "weather".to_string(), quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -351,7 +344,8 @@ fn test_duckdb_install() { Statement::Install { extension_name: Ident { value: "tpch".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() } } ); @@ -364,7 +358,8 @@ fn test_duckdb_load_extension() { Statement::Load { extension_name: Ident { value: "my_extension".to_string(), - quote_style: None + quote_style: None, + span: Span::empty() } }, stmt diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 0223e2915..a67399251 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -22,6 +22,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::{Span, Token, TokenWithLocation}; use test_utils::*; use sqlparser::ast::DataType::{Int, Text}; @@ -112,6 +113,7 @@ fn parse_create_procedure() { settings: None, format_clause: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("1")))], @@ -136,14 +138,16 @@ fn parse_create_procedure() { ProcedureParam { name: Ident { value: "@foo".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, data_type: DataType::Int(None) }, ProcedureParam { name: Ident { value: "@bar".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, data_type: DataType::Varchar(Some(CharacterLength::IntegerLength { length: 256, @@ -153,7 +157,8 @@ fn parse_create_procedure() { ]), name: ObjectName(vec![Ident { value: "test".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) } ) @@ -265,7 +270,8 @@ fn parse_mssql_create_role() { authorization_owner, Some(ObjectName(vec![Ident { value: "helena".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) ); } @@ -281,12 +287,14 @@ fn parse_alter_role() { [Statement::AlterRole { name: Ident { value: "old_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::RenameRole { role_name: Ident { value: "new_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, }] @@ -298,12 +306,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::AddMember { member_name: Ident { value: "new_member".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -315,12 +325,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::DropMember { member_name: Ident { value: "old_member".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -512,12 +524,14 @@ fn parse_substring_in_select() { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: Some(Distinct::Distinct), top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), substring_from: Some(Box::new(Expr::Value(number("0")))), substring_for: Some(Box::new(Expr::Value(number("1")))), @@ -528,7 +542,8 @@ fn parse_substring_in_select() { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "test".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -581,7 +596,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@foo".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: None, assignment: None, @@ -595,7 +611,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@bar".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: Some(Int(None)), assignment: None, @@ -609,7 +626,8 @@ fn parse_mssql_declare() { Declare { names: vec![Ident { value: "@baz".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }], data_type: Some(Text), assignment: Some(MsSqlAssignment(Box::new(Expr::Value(SingleQuotedString( @@ -670,10 +688,12 @@ fn parse_create_table_with_valid_options() { key: Ident { value: "DISTRIBUTION".to_string(), quote_style: None, + span: Span::empty(), }, value: Expr::Identifier(Ident { value: "ROUND_ROBIN".to_string(), quote_style: None, + span: Span::empty(), }) }, SqlOption::Partition { @@ -717,6 +737,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, asc: Some(true), }, @@ -724,6 +745,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, asc: Some(false), }, @@ -731,6 +753,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_c".to_string(), quote_style: None, + span: Span::empty(), }, asc: None, }, @@ -744,6 +767,7 @@ fn parse_create_table_with_valid_options() { key: Ident { value: "DISTRIBUTION".to_string(), quote_style: None, + span: Span::empty(), }, value: Expr::Function( Function { @@ -752,6 +776,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "HASH".to_string(), quote_style: None, + span: Span::empty(), }, ], ), @@ -766,6 +791,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, ), ), @@ -776,6 +802,7 @@ fn parse_create_table_with_valid_options() { Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, ), ), @@ -810,12 +837,14 @@ fn parse_create_table_with_valid_options() { name: ObjectName(vec![Ident { value: "mytable".to_string(), quote_style: None, + span: Span::empty(), },],), columns: vec![ ColumnDef { name: Ident { value: "column_a".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, @@ -825,6 +854,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_b".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, @@ -834,6 +864,7 @@ fn parse_create_table_with_valid_options() { name: Ident { value: "column_c".to_string(), quote_style: None, + span: Span::empty(), }, data_type: Int(None,), collation: None, diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index db5b9ec8d..a284b8ee5 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -25,7 +25,9 @@ use sqlparser::ast::MysqlInsertPriority::{Delayed, HighPriority, LowPriority}; use sqlparser::ast::*; use sqlparser::dialect::{GenericDialect, MySqlDialect}; use sqlparser::parser::{ParserError, ParserOptions}; +use sqlparser::tokenizer::Span; use sqlparser::tokenizer::Token; +use sqlparser::tokenizer::TokenWithLocation; use test_utils::*; #[macro_use] @@ -142,16 +144,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -179,16 +184,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -205,16 +213,19 @@ fn parse_flush() { ObjectName(vec![ Ident { value: "mek".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), }, Ident { value: "table1".to_string(), - quote_style: Some('`') + quote_style: Some('`'), + span: Span::empty(), } ]), ObjectName(vec![Ident { value: "table2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]) ] } @@ -948,11 +959,13 @@ fn parse_escaped_quote_identifiers_with_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted ` identifier".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -998,11 +1011,14 @@ fn parse_escaped_quote_identifiers_with_no_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "quoted `` identifier".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1041,11 +1057,14 @@ fn parse_escaped_backticks_with_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "`quoted identifier`".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1088,11 +1107,14 @@ fn parse_escaped_backticks_with_no_escape() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "``quoted identifier``".into(), quote_style: Some('`'), + span: Span::empty(), }))], into: None, from: vec![], @@ -1732,6 +1754,8 @@ fn parse_select_with_numeric_prefix_column_name() { assert_eq!( q.body, Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Identifier(Ident::new( @@ -1786,6 +1810,8 @@ fn parse_select_with_concatenation_of_exp_number_and_numeric_prefix_column() { assert_eq!( q.body, Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![ @@ -1934,7 +1960,8 @@ fn parse_delete_with_order_by() { vec![OrderByExpr { expr: Expr::Identifier(Ident { value: "id".to_owned(), - quote_style: None + quote_style: None, + span: Span::empty(), }), asc: Some(false), nulls_first: None, @@ -2015,7 +2042,8 @@ fn parse_alter_table_add_column() { }, column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), - quote_style: None + quote_style: None, + span: Span::empty(), })), },] ); @@ -2066,6 +2094,7 @@ fn parse_alter_table_add_columns() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }, ] @@ -2126,6 +2155,7 @@ fn parse_alter_table_change_column() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }; let sql4 = "ALTER TABLE orders CHANGE COLUMN description desc TEXT NOT NULL AFTER foo"; @@ -2165,6 +2195,7 @@ fn parse_alter_table_change_column_with_column_position() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("total_count"), quote_style: None, + span: Span::empty(), })), }; @@ -2221,6 +2252,7 @@ fn parse_alter_table_modify_column() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("foo"), quote_style: None, + span: Span::empty(), })), }; let sql4 = "ALTER TABLE orders MODIFY COLUMN description TEXT NOT NULL AFTER foo"; @@ -2258,6 +2290,7 @@ fn parse_alter_table_modify_column_with_column_position() { column_position: Some(MySQLColumnPosition::After(Ident { value: String::from("total_count"), quote_style: None, + span: Span::empty(), })), }; @@ -2276,6 +2309,8 @@ fn parse_alter_table_modify_column_with_column_position() { #[test] fn parse_substring_in_select() { + use sqlparser::tokenizer::Span; + let sql = "SELECT DISTINCT SUBSTRING(description, 0, 1) FROM test"; match mysql().one_statement_parses_to( sql, @@ -2286,12 +2321,14 @@ fn parse_substring_in_select() { Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: Some(Distinct::Distinct), top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Substring { expr: Box::new(Expr::Identifier(Ident { value: "description".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), substring_from: Some(Box::new(Expr::Value(number("0")))), substring_for: Some(Box::new(Expr::Value(number("1")))), @@ -2302,7 +2339,8 @@ fn parse_substring_in_select() { relation: TableFactor::Table { name: ObjectName(vec![Ident { value: "test".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), alias: None, args: None, @@ -2607,6 +2645,8 @@ fn parse_hex_string_introducer() { Statement::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), + distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::IntroducedString { diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index b9b3811ba..473658dba 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -21,6 +21,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::{Span, Token, TokenWithLocation}; use test_utils::*; use sqlparser::ast::*; @@ -1163,6 +1164,7 @@ fn parse_copy_to() { source: CopySource::Query(Box::new(Query { with: None, body: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![ @@ -1171,6 +1173,7 @@ fn parse_copy_to() { alias: Ident { value: "a".into(), quote_style: None, + span: Span::empty(), }, }, SelectItem::ExprWithAlias { @@ -1178,6 +1181,7 @@ fn parse_copy_to() { alias: Ident { value: "b".into(), quote_style: None, + span: Span::empty(), }, } ], @@ -1317,7 +1321,8 @@ fn parse_set() { variables: OneOrManyWithParens::One(ObjectName(vec![Ident::new("a")])), value: vec![Expr::Identifier(Ident { value: "b".into(), - quote_style: None + quote_style: None, + span: Span::empty(), })], } ); @@ -1353,7 +1358,8 @@ fn parse_set() { variables: OneOrManyWithParens::One(ObjectName(vec![Ident::new("a")])), value: vec![Expr::Identifier(Ident { value: "DEFAULT".into(), - quote_style: None + quote_style: None, + span: Span::empty(), })], } ); @@ -1382,7 +1388,8 @@ fn parse_set() { ])), value: vec![Expr::Identifier(Ident { value: "b".into(), - quote_style: None + quote_style: None, + span: Span::empty(), })], } ); @@ -1454,6 +1461,7 @@ fn parse_set_role() { role_name: Some(Ident { value: "rolename".to_string(), quote_style: Some('\"'), + span: Span::empty(), }), } ); @@ -1468,6 +1476,7 @@ fn parse_set_role() { role_name: Some(Ident { value: "rolename".to_string(), quote_style: Some('\''), + span: Span::empty(), }), } ); @@ -1764,7 +1773,8 @@ fn parse_pg_on_conflict() { selection: Some(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "dsize".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) @@ -1801,7 +1811,8 @@ fn parse_pg_on_conflict() { selection: Some(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident { value: "dsize".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Gt, right: Box::new(Expr::Value(Value::Placeholder("$2".to_string()))) @@ -2104,14 +2115,16 @@ fn parse_array_index_expr() { subscript: Box::new(Subscript::Index { index: Expr::Identifier(Ident { value: "baz".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }) }) }), subscript: Box::new(Subscript::Index { index: Expr::Identifier(Ident { value: "fooz".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }) }) }, @@ -2503,6 +2516,7 @@ fn parse_array_subquery_expr() { op: SetOperator::Union, set_quantifier: SetQuantifier::None, left: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("1")))], @@ -2523,6 +2537,7 @@ fn parse_array_subquery_expr() { connect_by: None, }))), right: Box::new(SetExpr::Select(Box::new(Select { + select_token: TokenWithLocation::wrap(Token::make_keyword("SELECT")), distinct: None, top: None, projection: vec![SelectItem::UnnamedExpr(Expr::Value(number("2")))], @@ -3182,6 +3197,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec![ "database".into(), @@ -3201,6 +3217,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec!["pg_catalog".into(), "~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) @@ -3216,6 +3233,7 @@ fn parse_custom_operator() { left: Box::new(Expr::Identifier(Ident { value: "relname".into(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::PGCustomBinaryOperator(vec!["~".into()]), right: Box::new(Expr::Value(Value::SingleQuotedString("^(table)$".into()))) @@ -3366,12 +3384,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "old_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::RenameRole { role_name: Ident { value: "new_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), } }, } @@ -3383,7 +3403,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::WithOptions { options: vec![ @@ -3412,7 +3433,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::WithOptions { options: vec![ @@ -3435,12 +3457,14 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::FromCurrent, in_database: None @@ -3454,17 +3478,20 @@ fn parse_alter_role() { [Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Value(Expr::Value(number("100000"))), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, }] @@ -3476,17 +3503,20 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Value(Expr::Value(number("100000"))), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3498,17 +3528,20 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Set { config_name: ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), config_value: SetConfigValue::Default, in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3520,7 +3553,8 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Reset { config_name: ResetConfig::ALL, @@ -3535,16 +3569,19 @@ fn parse_alter_role() { Statement::AlterRole { name: Ident { value: "role_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }, operation: AlterRoleOperation::Reset { config_name: ResetConfig::ConfigName(ObjectName(vec![Ident { value: "maintenance_work_mem".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])), in_database: Some(ObjectName(vec![Ident { value: "database_name".into(), - quote_style: None + quote_style: None, + span: Span::empty(), }])) }, } @@ -3688,7 +3725,8 @@ fn parse_drop_function() { func_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_func".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: None }], @@ -3704,7 +3742,8 @@ fn parse_drop_function() { func_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_func".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3729,7 +3768,8 @@ fn parse_drop_function() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_func1".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3747,7 +3787,8 @@ fn parse_drop_function() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_func2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Varchar(None)), @@ -3778,7 +3819,8 @@ fn parse_drop_procedure() { proc_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: None }], @@ -3794,7 +3836,8 @@ fn parse_drop_procedure() { proc_desc: vec![FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3819,7 +3862,8 @@ fn parse_drop_procedure() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc1".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Integer(None)), @@ -3837,7 +3881,8 @@ fn parse_drop_procedure() { FunctionDesc { name: ObjectName(vec![Ident { value: "test_proc2".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), args: Some(vec![ OperateFunctionArg::with_name("a", DataType::Varchar(None)), @@ -3918,6 +3963,7 @@ fn parse_dollar_quoted_string() { alias: Ident { value: "col_name".into(), quote_style: None, + span: Span::empty(), }, } ); @@ -4262,20 +4308,24 @@ fn test_simple_postgres_insert_with_alias() { into: true, table_name: ObjectName(vec![Ident { value: "test_tables".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), table_alias: Some(Ident { value: "test_table".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }), columns: vec![ Ident { value: "id".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, Ident { value: "a".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), } ], overwrite: false, @@ -4286,7 +4336,8 @@ fn test_simple_postgres_insert_with_alias() { rows: vec![vec![ Expr::Identifier(Ident { value: "DEFAULT".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }), Expr::Value(Value::Number("123".to_string(), false)) ]] @@ -4396,20 +4447,24 @@ fn test_simple_insert_with_quoted_alias() { into: true, table_name: ObjectName(vec![Ident { value: "test_tables".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }]), table_alias: Some(Ident { value: "Test_Table".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }), columns: vec![ Ident { value: "id".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }, Ident { value: "a".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), } ], overwrite: false, @@ -4420,7 +4475,8 @@ fn test_simple_insert_with_quoted_alias() { rows: vec![vec![ Expr::Identifier(Ident { value: "DEFAULT".to_string(), - quote_style: None + quote_style: None, + span: Span::empty(), }), Expr::Value(Value::SingleQuotedString("0123".to_string())) ]] @@ -5084,6 +5140,7 @@ fn check_arrow_precedence(sql: &str, arrow_operator: BinaryOperator) { left: Box::new(Expr::Identifier(Ident { value: "foo".to_string(), quote_style: None, + span: Span::empty(), })), op: arrow_operator, right: Box::new(Expr::Value(Value::SingleQuotedString("bar".to_string()))), @@ -5114,6 +5171,7 @@ fn arrow_cast_precedence() { left: Box::new(Expr::Identifier(Ident { value: "foo".to_string(), quote_style: None, + span: Span::empty(), })), op: BinaryOperator::Arrow, right: Box::new(Expr::Cast { diff --git a/tests/sqlparser_redshift.rs b/tests/sqlparser_redshift.rs index a25d50605..52f76e461 100644 --- a/tests/sqlparser_redshift.rs +++ b/tests/sqlparser_redshift.rs @@ -18,6 +18,7 @@ #[macro_use] mod test_utils; +use sqlparser::tokenizer::Span; use test_utils::*; use sqlparser::ast::*; @@ -31,7 +32,8 @@ fn test_square_brackets_over_db_schema_table_name() { select.projection[0], SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "col1".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), })), ); assert_eq!( @@ -41,11 +43,13 @@ fn test_square_brackets_over_db_schema_table_name() { name: ObjectName(vec![ Ident { value: "test_schema".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), }, Ident { value: "test_table".to_string(), - quote_style: Some('[') + quote_style: Some('['), + span: Span::empty(), } ]), alias: None, @@ -78,7 +82,8 @@ fn test_double_quotes_over_db_schema_table_name() { select.projection[0], SelectItem::UnnamedExpr(Expr::Identifier(Ident { value: "col1".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), })), ); assert_eq!( @@ -88,11 +93,13 @@ fn test_double_quotes_over_db_schema_table_name() { name: ObjectName(vec![ Ident { value: "test_schema".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), }, Ident { value: "test_table".to_string(), - quote_style: Some('"') + quote_style: Some('"'), + span: Span::empty(), } ]), alias: None,