Change macro syntax to be token tree based and fix legacy macros. (#6388

)

corelib/src/test/language_features/for_test.cairo

@@ -22,7 +22,7 @@ fn test_for_loop_array_variables() {
 fn test_for_loop_array_tuples() {
     let mut i = 10;
     for (x, y) in array![
-        (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17)
+        (10, 10), (11, 11), (12, 12), (13, 13), (14, 14), (15, 15), (16, 16), (17, 17),
     ] {
         assert_eq!(x, i);
         assert_eq!(y, i);

crates/cairo-lang-defs/src/diagnostic_utils.rs

@@ -3,39 +3,62 @@ use std::fmt;
 use cairo_lang_debug::DebugWithDb;
 use cairo_lang_diagnostics::DiagnosticLocation;
 use cairo_lang_filesystem::ids::FileId;
-use cairo_lang_filesystem::span::TextSpan;
+use cairo_lang_filesystem::span::{TextSpan, TextWidth};
 use cairo_lang_syntax::node::ids::SyntaxStablePtrId;
 use cairo_lang_syntax::node::{SyntaxNode, TypedSyntaxNode};
 
 use crate::db::DefsGroup;
 
 /// A stable location of a real, concrete syntax.
 #[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
-pub struct StableLocation(SyntaxStablePtrId);
+pub struct StableLocation {
+    stable_ptr: SyntaxStablePtrId,
+    /// An optional inner span of the stable location. Useful for diagnostics caused by inline
+    /// macros, see [crate::plugin::PluginDiagnostic] for more information. The tuple is (offset,
+    /// width).
+    inner_span: Option<(TextWidth, TextWidth)>,
+}
+
 impl StableLocation {
     pub fn new(stable_ptr: SyntaxStablePtrId) -> Self {
-        Self(stable_ptr)
+        Self { stable_ptr, inner_span: None }
+    }
+
+    pub fn with_inner_span(
+        stable_ptr: SyntaxStablePtrId,
+        inner_span: (TextWidth, TextWidth),
+    ) -> Self {
+        Self { stable_ptr, inner_span: Some(inner_span) }
     }
 
     pub fn file_id(&self, db: &dyn DefsGroup) -> FileId {
-        self.0.file_id(db.upcast())
+        self.stable_ptr.file_id(db.upcast())
     }
 
     pub fn from_ast<TNode: TypedSyntaxNode>(node: &TNode) -> Self {
-        Self(node.as_syntax_node().stable_ptr())
+        Self::new(node.as_syntax_node().stable_ptr())
     }
 
     /// Returns the [SyntaxNode] that corresponds to the [StableLocation].
     pub fn syntax_node(&self, db: &dyn DefsGroup) -> SyntaxNode {
-        self.0.lookup(db.upcast())
+        self.stable_ptr.lookup(db.upcast())
     }
 
     /// Returns the [DiagnosticLocation] that corresponds to the [StableLocation].
     pub fn diagnostic_location(&self, db: &dyn DefsGroup) -> DiagnosticLocation {
-        let syntax_node = self.syntax_node(db);
-        DiagnosticLocation {
-            file_id: self.file_id(db),
-            span: syntax_node.span_without_trivia(db.upcast()),
+        match self.inner_span {
+            Some((start, width)) => {
+                let start = self.syntax_node(db).offset().add_width(start);
+                let end = start.add_width(width);
+                DiagnosticLocation { file_id: self.file_id(db), span: TextSpan { start, end } }
+            }
+            None => {
+                let syntax_node = self.syntax_node(db);
+                DiagnosticLocation {
+                    file_id: self.file_id(db),
+                    span: syntax_node.span_without_trivia(db.upcast()),
+                }
+            }
         }
     }
 
@@ -46,9 +69,13 @@ impl StableLocation {
         until_stable_ptr: SyntaxStablePtrId,
     ) -> DiagnosticLocation {
         let syntax_db = db.upcast();
-        let start = self.0.lookup(syntax_db).span_start_without_trivia(syntax_db);
+        let start = self.stable_ptr.lookup(syntax_db).span_start_without_trivia(syntax_db);
         let end = until_stable_ptr.lookup(syntax_db).span_end_without_trivia(syntax_db);
-        DiagnosticLocation { file_id: self.0.file_id(syntax_db), span: TextSpan { start, end } }
+
+        DiagnosticLocation {
+            file_id: self.stable_ptr.file_id(syntax_db),
+            span: TextSpan { start, end },
+        }
     }
 }
 

crates/cairo-lang-defs/src/plugin.rs

@@ -6,9 +6,10 @@ use cairo_lang_diagnostics::Severity;
 use cairo_lang_filesystem::cfg::CfgSet;
 use cairo_lang_filesystem::db::Edition;
 use cairo_lang_filesystem::ids::CodeMapping;
-use cairo_lang_syntax::node::ast;
+use cairo_lang_filesystem::span::TextWidth;
 use cairo_lang_syntax::node::db::SyntaxGroup;
 use cairo_lang_syntax::node::ids::SyntaxStablePtrId;
+use cairo_lang_syntax::node::{SyntaxNode, ast};
 use cairo_lang_utils::ordered_hash_set::OrderedHashSet;
 use smol_str::SmolStr;
 
@@ -63,18 +64,56 @@ pub struct PluginResult {
     pub remove_original_item: bool,
 }
 
+/// A diagnostic generated by a plugin.
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct PluginDiagnostic {
+    /// The stable pointer of the syntax node that caused the diagnostic.
     pub stable_ptr: SyntaxStablePtrId,
+    /// The content of the diagnostic.
     pub message: String,
+    /// The severity of the diagnostic.
     pub severity: Severity,
+    /// An optional inner span inside the stable pointer that caused the diagnostic. Useful for
+    /// diagnostics caused by inline macros, since the syntax of the arguments is a token tree and
+    /// is not segmented into each argument.
+    /// The tuple is (offset, width).
+    pub inner_span: Option<(TextWidth, TextWidth)>,
 }
 impl PluginDiagnostic {
     pub fn error(stable_ptr: impl Into<SyntaxStablePtrId>, message: String) -> PluginDiagnostic {
-        PluginDiagnostic { stable_ptr: stable_ptr.into(), message, severity: Severity::Error }
+        PluginDiagnostic {
+            stable_ptr: stable_ptr.into(),
+            message,
+            severity: Severity::Error,
+            inner_span: None,
+        }
     }
+
+    /// Creates a diagnostic, pointing to an inner span inside the given stable pointer.
+    pub fn error_with_inner_span(
+        db: &dyn SyntaxGroup,
+        stable_ptr: impl Into<SyntaxStablePtrId>,
+        inner_span: SyntaxNode,
+        message: String,
+    ) -> PluginDiagnostic {
+        let stable_ptr = stable_ptr.into();
+        let offset = inner_span.offset() - stable_ptr.lookup(db).offset();
+        let width = inner_span.width(db);
+        PluginDiagnostic {
+            stable_ptr,
+            message,
+            severity: Severity::Error,
+            inner_span: Some((offset, width)),
+        }
+    }
+
     pub fn warning(stable_ptr: impl Into<SyntaxStablePtrId>, message: String) -> PluginDiagnostic {
-        PluginDiagnostic { stable_ptr: stable_ptr.into(), message, severity: Severity::Warning }
+        PluginDiagnostic {
+            stable_ptr: stable_ptr.into(),
+            message,
+            severity: Severity::Warning,
+            inner_span: None,
+        }
     }
 }
 

crates/cairo-lang-defs/src/plugin_utils.rs

@@ -1,5 +1,6 @@
 use cairo_lang_syntax::node::db::SyntaxGroup;
 use cairo_lang_syntax::node::helpers::WrappedArgListHelper;
+use cairo_lang_syntax::node::ids::SyntaxStablePtrId;
 use cairo_lang_syntax::node::{SyntaxNode, TypedSyntaxNode, ast};
 use cairo_lang_utils::require;
 use itertools::Itertools;
@@ -14,7 +15,7 @@ pub trait InlineMacroCall {
     fn path(&self, db: &dyn SyntaxGroup) -> Self::PathNode;
 }
 
-impl InlineMacroCall for ast::ExprInlineMacro {
+impl InlineMacroCall for ast::LegacyExprInlineMacro {
     type PathNode = ast::ExprPath;
     type Result = InlinePluginResult;
 
@@ -27,7 +28,7 @@ impl InlineMacroCall for ast::ExprInlineMacro {
     }
 }
 
-impl InlineMacroCall for ast::ItemInlineMacro {
+impl InlineMacroCall for ast::LegacyItemInlineMacro {
     type PathNode = ast::TerminalIdentifier;
     type Result = PluginResult;
 
@@ -60,17 +61,29 @@ impl PluginResultTrait for PluginResult {
 /// Returns diagnostics for an unsupported bracket type.
 pub fn unsupported_bracket_diagnostic<CallAst: InlineMacroCall>(
     db: &dyn SyntaxGroup,
-    macro_ast: &CallAst,
+    legacy_macro_ast: &CallAst,
+    macro_ast: impl Into<SyntaxStablePtrId>,
 ) -> CallAst::Result {
-    CallAst::Result::diagnostic_only(PluginDiagnostic::error(
-        macro_ast.arguments(db).left_bracket_stable_ptr(db),
+    CallAst::Result::diagnostic_only(PluginDiagnostic::error_with_inner_span(
+        db,
+        macro_ast,
+        legacy_macro_ast.arguments(db).left_bracket_syntax_node(db),
         format!(
             "Macro `{}` does not support this bracket type.",
-            macro_ast.path(db).as_syntax_node().get_text_without_trivia(db)
+            legacy_macro_ast.path(db).as_syntax_node().get_text_without_trivia(db)
         ),
     ))
 }
 
+pub fn not_legacy_macro_diagnostic(stable_ptr: SyntaxStablePtrId) -> PluginDiagnostic {
+    PluginDiagnostic::error(
+        stable_ptr,
+        "Macro can not be parsed as legacy macro. Expected an argument list wrapped in either \
+         parentheses, brackets, or braces."
+            .to_string(),
+    )
+}
+
 /// Extracts a single unnamed argument.
 pub fn extract_single_unnamed_arg(
     db: &dyn SyntaxGroup,
@@ -118,14 +131,19 @@ pub fn escape_node(db: &dyn SyntaxGroup, node: SyntaxNode) -> String {
 ///     db,
 ///     syntax,
 ///     2,
-///     ast::WrappedArgList::ParenthesizedArgList(_) | ast::WrappedArgList::BracedArgList(_)
+///     ast::WrappedArgList::ParenthesizedArgList(_) | ast::WrappedArgList::BracedArgList(_),
+///     token_tree_syntax.stable_ptr()
 /// );
 #[macro_export]
 macro_rules! extract_macro_unnamed_args {
-    ($db:expr, $syntax:expr, $n:expr, $pattern:pat) => {{
+    ($db:expr, $syntax:expr, $n:expr, $pattern:pat, $diagnostics_ptr:expr) => {{
         let arguments = $crate::plugin_utils::InlineMacroCall::arguments($syntax, $db);
         if !matches!(arguments, $pattern) {
-            return $crate::plugin_utils::unsupported_bracket_diagnostic($db, $syntax);
+            return $crate::plugin_utils::unsupported_bracket_diagnostic(
+                $db,
+                $syntax,
+                $diagnostics_ptr,
+            );
         }
         // `unwrap` is ok because the above `matches` condition ensures it's not None (unless
         // the pattern contains the `Missing` variant).
@@ -137,7 +155,7 @@ macro_rules! extract_macro_unnamed_args {
         let Some(args) = args else {
             return $crate::plugin_utils::PluginResultTrait::diagnostic_only(
                 PluginDiagnostic::error(
-                    $syntax,
+                    $diagnostics_ptr,
                     format!(
                         "Macro `{}` must have exactly {} unnamed arguments.",
                         $crate::plugin_utils::InlineMacroCall::path($syntax, $db)
@@ -154,18 +172,23 @@ macro_rules! extract_macro_unnamed_args {
 }
 
 /// Macro to extract a single unnamed argument of an inline macro.
+///
 /// Gets the pattern for the allowed bracket types, and returns the argument expression.
+/// The arguments are extracted from a `WrappedArgList` node syntax, as was in legacy inline macros.
+/// However, as macros are now parsed as general token trees, the diagnostics pointer is passed to
+/// the macro to allow pointing to the original location.
 ///
 /// Example usage (allowing `()` or `{}` brackets):
 /// let arg = extract_macro_single_unnamed_arg!(
 ///     db,
-///     syntax,
-///     ast::WrappedArgList::ParenthesizedArgList(_) | ast::WrappedArgList::BracedArgList(_)
+///     arg_list_syntax,
+///     ast::WrappedArgList::ParenthesizedArgList(_) | ast::WrappedArgList::BracedArgList(_),
+///     token_tree_syntax.stable_ptr()
 /// );
 #[macro_export]
 macro_rules! extract_macro_single_unnamed_arg {
-    ($db:expr, $syntax:expr, $pattern:pat) => {{
-        let [x] = $crate::extract_macro_unnamed_args!($db, $syntax, 1, $pattern);
+    ($db:expr, $syntax:expr, $pattern:pat, $diagnostics_ptr:expr) => {{
+        let [x] = $crate::extract_macro_unnamed_args!($db, $syntax, 1, $pattern, $diagnostics_ptr);
         x
     }};
 }

crates/cairo-lang-defs/src/test.rs

@@ -476,6 +476,6 @@ fn test_unknown_item_macro() {
         format!("{:?}", db.module_plugin_diagnostics(module_id).unwrap()),
         "[(ModuleFileId(CrateRoot(CrateId(0)), FileIndex(0)), PluginDiagnostic { stable_ptr: \
          SyntaxStablePtrId(3), message: \"Unknown inline item macro: 'unknown_item_macro'.\", \
-         severity: Error })]"
+         severity: Error, inner_span: None })]"
     )
 }

crates/cairo-lang-formatter/src/formatter_impl.rs

@@ -2,9 +2,10 @@ use std::cmp::Ordering;
 use std::fmt;
 
 use cairo_lang_filesystem::span::TextWidth;
+use cairo_lang_parser::macro_helpers::token_tree_as_wrapped_arg_list;
 use cairo_lang_syntax as syntax;
 use cairo_lang_syntax::attribute::consts::FMT_SKIP_ATTR;
-use cairo_lang_syntax::node::ast::UsePath;
+use cairo_lang_syntax::node::ast::{TokenTreeNode, UsePath};
 use cairo_lang_syntax::node::db::SyntaxGroup;
 use cairo_lang_syntax::node::{SyntaxNode, Terminal, TypedSyntaxNode, ast};
 use itertools::Itertools;
@@ -792,6 +793,24 @@ impl<'a> FormatterImpl<'a> {
     /// Appends a formatted string, representing the syntax_node, to the result.
     /// Should be called with a root syntax node to format a file.
     fn format_node(&mut self, syntax_node: &SyntaxNode) {
+        // If we encounter a token tree node, i.e. a macro, we try to parse it as a
+        // [ast::WrappedArgList] (the syntax kind of legacy macro calls). If successful, we
+        // format the wrapped arg list according to the rules of wrapped arg lists, otherwise we
+        // treat it as a normal syntax node, and in practice no formatting is done.
+        // TODO(Gil): Consider if we want to keep this behavior when general macro support is added.
+        if syntax_node.kind(self.db) == SyntaxKind::TokenTreeNode {
+            let as_wrapped_arg_list = token_tree_as_wrapped_arg_list(
+                TokenTreeNode::from_syntax_node(self.db, syntax_node.clone()),
+                self.db,
+            );
+            let file_id = syntax_node.stable_ptr().file_id(self.db);
+
+            if let Some(wrapped_arg_list) = as_wrapped_arg_list {
+                let new_syntax_node = SyntaxNode::new_root(self.db, file_id, wrapped_arg_list.0);
+                self.format_node(&new_syntax_node);
+                return;
+            }
+        }
         if syntax_node.text(self.db).is_some() {
             panic!("Token reached before terminal.");
         }

crates/cairo-lang-formatter/src/node_properties.rs

@@ -378,7 +378,10 @@ impl SyntaxNodeFormat for SyntaxNode {
                 | SyntaxKind::TraitItemList
                 | SyntaxKind::ImplItemList
                 | SyntaxKind::UsePathMulti
-                | SyntaxKind::ItemEnum => Some(5),
+                | SyntaxKind::ItemEnum
+                | SyntaxKind::ParenthesizedTokenTree
+                | SyntaxKind::BracedTokenTree
+                | SyntaxKind::BracketedTokenTree => Some(5),
                 _ => None,
             },
         }

crates/cairo-lang-language-server/src/ide/macros/expand.rs

@@ -155,6 +155,7 @@ fn expand_inline_macros(
         &mut files,
         &mut output,
         FileProcessorConfig::main_file(db, node_to_expand, top_level_macro_kind),
+        top_level_macro_kind,
     )?;
 
     while let Some(file) = files.pop_front() {
@@ -165,6 +166,7 @@ fn expand_inline_macros(
             &mut files,
             &mut output,
             FileProcessorConfig::generated_file(db, file, db.file_content(file)?.to_string())?,
+            top_level_macro_kind,
         )?;
     }
 
@@ -250,6 +252,7 @@ fn expand_inline_macros_in_single_file(
     files: &mut VecDeque<FileId>,
     output: &mut String,
     mut config: FileProcessorConfig,
+    top_level_macro_kind: TopLevelMacroKind,
 ) -> Option<()> {
     let plugins = db.inline_macro_plugins();
 
@@ -277,10 +280,12 @@ fn expand_inline_macros_in_single_file(
             name: file.file_name(db).into(),
             content: config.content.into(),
             code_mappings: Default::default(),
-            kind: file.kind(db),
+            kind: match top_level_macro_kind {
+                TopLevelMacroKind::Inline => FileKind::Expr,
+                TopLevelMacroKind::Attribute => FileKind::Module,
+            },
         })
         .intern(db);
-
         files.push_back(new_file);
     };