Auto merge of rust-lang#119412 - petrochenkov:dialoc3, r=<try>

macro_rules: Remove `NtIdent` nonterminal token

It only exists for keeping the second span (the metavariable span), but after rust-lang#119673 such spans can be kept using a more general mechanism.

compiler/rustc_ast/src/mut_visit.rs

@@ -766,9 +766,6 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
             *span = ident.span;
             return; // Avoid visiting the span for the second time.
         }
-        token::NtIdent(ident, _is_raw) => {
-            vis.visit_ident(ident);
-        }
         token::NtLifetime(ident) => {
             vis.visit_ident(ident);
         }

compiler/rustc_ast/src/token.rs

@@ -317,14 +317,7 @@ pub enum TokenKind {
     Literal(Lit),
 
     /// Identifier token.
-    /// Do not forget about `NtIdent` when you want to match on identifiers.
-    /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated identifiers in the same way.
     Ident(Symbol, IdentIsRaw),
-    /// This identifier (and its span) is the identifier passed to the
-    /// declarative macro. The span in the surrounding `Token` is the span of
-    /// the `ident` metavariable in the macro's RHS.
-    NtIdent(Ident, IdentIsRaw),
 
     /// Lifetime identifier token.
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
@@ -457,7 +450,7 @@ impl Token {
     /// if they keep spans or perform edition checks.
     pub fn uninterpolated_span(&self) -> Span {
         match self.kind {
-            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
+            NtLifetime(ident) => ident.span,
             Interpolated(ref nt) => nt.use_span(),
             _ => self.span,
         }
@@ -476,7 +469,7 @@ impl Token {
             }
 
             OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
-            | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
+            | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
         }
     }
 
@@ -644,7 +637,6 @@ impl Token {
     /// otherwise returns the original token.
     pub fn uninterpolate(&self) -> Cow<'_, Token> {
         match self.kind {
-            NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
             NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
             _ => Cow::Borrowed(self),
         }
@@ -656,7 +648,6 @@ impl Token {
         // We avoid using `Token::uninterpolate` here because it's slow.
         match self.kind {
             Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
-            NtIdent(ident, is_raw) => Some((ident, is_raw)),
             _ => None,
         }
     }
@@ -856,7 +847,7 @@ impl Token {
 
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
             | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
-            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
+            | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
             | Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
                 return None;
             }

compiler/rustc_ast/src/tokenstream.rs

@@ -484,9 +484,6 @@ impl TokenStream {
 
     fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
         match token.kind {
-            token::NtIdent(ident, is_raw) => {
-                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
-            }
             token::NtLifetime(ident) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
@@ -516,10 +513,9 @@ impl TokenStream {
     pub fn flattened(&self) -> TokenStream {
         fn can_skip(stream: &TokenStream) -> bool {
             stream.trees().all(|tree| match tree {
-                TokenTree::Token(token, _) => !matches!(
-                    token.kind,
-                    token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
-                ),
+                TokenTree::Token(token, _) => {
+                    !matches!(token.kind, token::NtLifetime(..) | token::Interpolated(..))
+                }
                 TokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -943,9 +943,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::Ident(name, is_raw) => {
                 IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
             }
-            token::NtIdent(ident, is_raw) => {
-                IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
-            }
             token::Lifetime(name) => name.to_string().into(),
             token::NtLifetime(ident) => ident.name.to_string().into(),
 

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -743,9 +743,11 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
         // or
         // `fn foo(&x: &i32)` -> `fn foo(&(mut x): &i32)`
         let def_id = self.body.source.def_id();
-        if let Some(local_def_id) = def_id.as_local()
-            && let Some(body) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
-            && let Some(hir_id) = (BindingFinder { span: pat_span }).visit_body(&body).break_value()
+        let hir_id = def_id
+            .as_local()
+            .and_then(|def_id| self.infcx.tcx.hir().maybe_body_owned_by(def_id))
+            .and_then(|body| (BindingFinder { span: pat_span }).visit_body(&body).break_value());
+        if let Some(hir_id) = hir_id
             && let node = self.infcx.tcx.hir_node(hir_id)
             && let hir::Node::LetStmt(hir::LetStmt {
                 pat: hir::Pat { kind: hir::PatKind::Ref(_, _), .. },
@@ -767,8 +769,11 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
             return;
         }
 
+        let span = local_decl.source_info.span;
+        let span =
+            hir_id.map_or(span, |hir_id| span.with_neighbor(self.infcx.tcx.hir().span(hir_id)));
         err.span_suggestion_verbose(
-            local_decl.source_info.span.shrink_to_lo(),
+            span.shrink_to_lo(),
             "consider changing this to be mutable",
             "mut ",
             Applicability::MachineApplicable,

compiler/rustc_expand/src/config.rs

@@ -203,13 +203,7 @@ impl<'a> StripUnconfigured<'a> {
                     Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
                 }
                 AttrTokenTree::Token(
-                    Token {
-                        kind:
-                            TokenKind::NtIdent(..)
-                            | TokenKind::NtLifetime(..)
-                            | TokenKind::Interpolated(..),
-                        ..
-                    },
+                    Token { kind: TokenKind::NtLifetime(..) | TokenKind::Interpolated(..), .. },
                     _,
                 ) => {
                     panic!("Nonterminal should have been flattened: {:?}", tree);

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -274,11 +274,6 @@ pub(super) fn transcribe<'a>(
                             // without wrapping them into groups.
                             maybe_use_metavar_location(psess, &stack, sp, tt, &mut marker)
                         }
-                        MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
-                            marker.visit_span(&mut sp);
-                            let kind = token::NtIdent(*ident, *is_raw);
-                            TokenTree::token_alone(kind, sp)
-                        }
                         MatchedSingle(ParseNtResult::Lifetime(ident)) => {
                             marker.visit_span(&mut sp);
                             let kind = token::NtLifetime(*ident);
@@ -395,9 +390,22 @@ fn maybe_use_metavar_location(
         return orig_tt.clone();
     }
 
-    let insert = |mspans: &mut FxHashMap<_, _>, s, ms| match mspans.try_insert(s, ms) {
+    let insert = |mspans: &mut FxHashMap<_, _>, s, ms: Span| match mspans.try_insert(s, ms) {
         Ok(_) => true,
-        Err(err) => *err.entry.get() == ms, // Tried to insert the same span, still success
+        Err(mut err) => {
+            let old_ms = *err.entry.get();
+            if ms == old_ms {
+                // Tried to insert the same span, still success.
+                return true;
+            }
+            if !ms.eq_ctxt(old_ms) && ms.ctxt().outer_expn_data().call_site.eq_ctxt(old_ms) {
+                // This looks like a variable passed to an inner (possibly recursive) macro call.
+                // The innermost metavar span is the most useful, so override.
+                err.entry.insert(ms);
+                return true;
+            }
+            false
+        }
     };
     marker.visit_span(&mut metavar_span);
     let no_collision = match orig_tt {
@@ -411,16 +419,23 @@ fn maybe_use_metavar_location(
         }),
     };
     if no_collision || psess.source_map().is_imported(metavar_span) {
+        // Add a whitespace for backward compatibility.
+        // FIXME: assign spacing to tokens from metavars in a more systematic way (#127123).
+        if let TokenTree::Token(token, _) = orig_tt {
+            return TokenTree::Token(token.clone(), Spacing::Alone);
+        }
         return orig_tt.clone();
     }
 
     // Setting metavar spans for the heuristic spans gives better opportunities for combining them
     // with neighboring spans even despite their different syntactic contexts.
     match orig_tt {
-        TokenTree::Token(Token { kind, span }, spacing) => {
+        TokenTree::Token(Token { kind, span }, _spacing) => {
             let span = metavar_span.with_ctxt(span.ctxt());
             with_metavar_spans(|mspans| insert(mspans, span, metavar_span));
-            TokenTree::Token(Token { kind: kind.clone(), span }, *spacing)
+            // Add a whitespace for backward compatibility.
+            // FIXME: assign spacing to tokens from metavars in a more systematic way (#127123).
+            TokenTree::Token(Token { kind: kind.clone(), span }, Spacing::Alone)
         }
         TokenTree::Delimited(dspan, dspacing, delimiter, tts) => {
             let open = metavar_span.with_ctxt(dspan.open.ctxt());
@@ -735,12 +750,6 @@ fn extract_ident<'a>(
     interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
 ) -> PResult<'a, String> {
     if let NamedMatch::MatchedSingle(pnr) = matched_from_ident(dcx, ident, interp)? {
-        if let ParseNtResult::Ident(nt_ident, is_raw) = pnr {
-            if let IdentIsRaw::Yes = is_raw {
-                return Err(dcx.struct_span_err(ident.span, RAW_IDENT_ERR));
-            }
-            return Ok(nt_ident.to_string());
-        }
         if let ParseNtResult::Tt(TokenTree::Token(
             Token { kind: TokenKind::Ident(token_ident, is_raw), .. },
             _,

compiler/rustc_expand/src/proc_macro_server.rs

@@ -221,11 +221,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                 Ident(sym, is_raw) => {
                     trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span }))
                 }
-                NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident {
-                    sym: ident.name,
-                    is_raw: is_raw.into(),
-                    span: ident.span,
-                })),
 
                 Lifetime(name) => {
                     let ident = symbol::Ident::new(name, span).without_first_quote();

compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs

@@ -56,7 +56,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             ));
         }
 
-        if self_ty.span.edition().at_least_rust_2021() {
+        if tcx.hir().span_in_context(self_ty.hir_id).edition().at_least_rust_2021() {
             let msg = "trait objects must include the `dyn` keyword";
             let label = "add `dyn` keyword before this trait";
             let mut diag =

compiler/rustc_hir_typeck/src/method/suggest.rs

@@ -2454,7 +2454,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 ExprKind::Path(QPath::Resolved(_, path)) => {
                     // local binding
                     if let hir::def::Res::Local(hir_id) = path.res {
-                        let span = tcx.hir().span(hir_id);
+                        let span = tcx.hir().span_in_context(hir_id);
                         let filename = tcx.sess.source_map().span_to_filename(span);
 
                         let parent_node = self.tcx.parent_hir_node(hir_id);

compiler/rustc_lint/src/types.rs

@@ -718,8 +718,9 @@ fn lint_wide_pointer<'tcx>(
         return;
     };
 
+    let (l_span, r_span) = (l.span.with_neighbor(e.span), r.span.with_neighbor(e.span));
     let (Some(l_span), Some(r_span)) =
-        (l.span.find_ancestor_inside(e.span), r.span.find_ancestor_inside(e.span))
+        (l_span.find_ancestor_inside(e.span), r_span.find_ancestor_inside(e.span))
     else {
         return cx.emit_span_lint(
             AMBIGUOUS_WIDE_POINTER_COMPARISONS,

compiler/rustc_middle/src/hir/map/mod.rs

@@ -935,6 +935,16 @@ impl<'hir> Map<'hir> {
         }
     }
 
+    /// Use regular `span` method when *showing* any diagnostics.
+    /// Use this method when suggesting *inserting* any code to the left or right from this node.
+    ///
+    /// If some piece of code is passed to a macro as an argument, then this method may move the
+    /// span from the argument into the macro body where that argument is emitted into the output.
+    pub fn span_in_context(self, hir_id: HirId) -> Span {
+        let parent_span = self.span(self.tcx.parent_hir_id(hir_id));
+        self.span(hir_id).with_neighbor(parent_span)
+    }
+
     /// Get a representation of this `id` for debugging purposes.
     /// NOTE: Do NOT use this in diagnostics!
     pub fn node_to_string(self, id: HirId) -> String {

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -230,8 +230,7 @@ impl<'a> Parser<'a> {
         let (mut ret, trailing) = ret?;
 
         // When we're not in `capture-cfg` mode, then bail out early if:
-        // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`)
-        //    so there's nothing for us to do.
+        // 1. Our target doesn't support tokens at all, so there's nothing for us to do.
         // 2. Our target already has tokens set (e.g. we've parsed something
         // like `#[my_attr] $item`. The actual parsing code takes care of prepending
         // any attributes to the nonterminal, so we don't need to modify the

compiler/rustc_parse/src/parser/expr.rs

@@ -249,10 +249,11 @@ impl<'a> Parser<'a> {
                 continue;
             }
 
+            let op_span = op.span;
             let op = op.node;
             // Special cases:
             if op == AssocOp::As {
-                lhs = self.parse_assoc_op_cast(lhs, lhs_span, ExprKind::Cast)?;
+                lhs = self.parse_assoc_op_cast(lhs, lhs_span, op_span, ExprKind::Cast)?;
                 continue;
             } else if op == AssocOp::DotDot || op == AssocOp::DotDotEq {
                 // If we didn't have to handle `x..`/`x..=`, it would be pretty easy to
@@ -274,7 +275,7 @@ impl<'a> Parser<'a> {
                 this.parse_expr_assoc_with(prec + prec_adjustment, LhsExpr::Unparsed { attrs })
             })?;
 
-            let span = self.mk_expr_sp(&lhs, lhs_span, rhs.span);
+            let span = self.mk_expr_sp(&lhs, lhs_span, op_span, rhs.span);
             lhs = match op {
                 AssocOp::Add
                 | AssocOp::Subtract
@@ -453,7 +454,7 @@ impl<'a> Parser<'a> {
             None
         };
         let rhs_span = rhs.as_ref().map_or(cur_op_span, |x| x.span);
-        let span = self.mk_expr_sp(&lhs, lhs.span, rhs_span);
+        let span = self.mk_expr_sp(&lhs, lhs.span, cur_op_span, rhs_span);
         let limits =
             if op == AssocOp::DotDot { RangeLimits::HalfOpen } else { RangeLimits::Closed };
         let range = self.mk_range(Some(lhs), rhs, limits);
@@ -663,9 +664,7 @@ impl<'a> Parser<'a> {
     /// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
     fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
         match self.prev_token.kind {
-            TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
-                self.prev_token.span
-            }
+            TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => self.prev_token.span,
             _ => expr.span,
         }
     }
@@ -674,10 +673,11 @@ impl<'a> Parser<'a> {
         &mut self,
         lhs: P<Expr>,
         lhs_span: Span,
+        op_span: Span,
         expr_kind: fn(P<Expr>, P<Ty>) -> ExprKind,
     ) -> PResult<'a, P<Expr>> {
         let mk_expr = |this: &mut Self, lhs: P<Expr>, rhs: P<Ty>| {
-            this.mk_expr(this.mk_expr_sp(&lhs, lhs_span, rhs.span), expr_kind(lhs, rhs))
+            this.mk_expr(this.mk_expr_sp(&lhs, lhs_span, op_span, rhs.span), expr_kind(lhs, rhs))
         };
 
         // Save the state of the parser before parsing type normally, in case there is a
@@ -3852,11 +3852,13 @@ impl<'a> Parser<'a> {
 
     /// Create expression span ensuring the span of the parent node
     /// is larger than the span of lhs and rhs, including the attributes.
-    fn mk_expr_sp(&self, lhs: &P<Expr>, lhs_span: Span, rhs_span: Span) -> Span {
+    fn mk_expr_sp(&self, lhs: &P<Expr>, lhs_span: Span, op_span: Span, rhs_span: Span) -> Span {
         lhs.attrs
             .iter()
             .find(|a| a.style == AttrStyle::Outer)
             .map_or(lhs_span, |a| a.span)
+            // An approximation to #126763.
+            .to(op_span)
             .to(rhs_span)
     }
 

compiler/rustc_parse/src/parser/mod.rs

@@ -394,7 +394,6 @@ pub(super) fn token_descr(token: &Token) -> String {
         (Some(TokenDescription::Keyword), _) => Some("keyword"),
         (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
         (Some(TokenDescription::DocComment), _) => Some("doc comment"),
-        (None, TokenKind::NtIdent(..)) => Some("identifier"),
         (None, TokenKind::NtLifetime(..)) => Some("lifetime"),
         (None, TokenKind::Interpolated(node)) => Some(node.descr()),
         (None, _) => None,
@@ -687,9 +686,9 @@ impl<'a> Parser<'a> {
         self.is_keyword_ahead(0, &[kw::Const])
             && self.look_ahead(1, |t| match &t.kind {
                 // async closures do not work with const closures, so we do not parse that here.
-                token::Ident(kw::Move | kw::Static, _) | token::OrOr | token::BinOp(token::Or) => {
-                    true
-                }
+                token::Ident(kw::Move | kw::Static, IdentIsRaw::No)
+                | token::OrOr
+                | token::BinOp(token::Or) => true,
                 _ => false,
             })
     }
@@ -1623,7 +1622,6 @@ enum FlatToken {
 #[derive(Clone, Debug)]
 pub enum ParseNtResult {
     Tt(TokenTree),
-    Ident(Ident, IdentIsRaw),
     Lifetime(Ident),
 
     /// This case will eventually be removed, along with `Token::Interpolate`.