diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index 4dc9c30a2c807..a28844b00b74b 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -231,29 +231,29 @@ impl HasTokens for Attribute { impl HasTokens for Nonterminal { fn tokens(&self) -> Option<&LazyAttrTokenStream> { match self { - Nonterminal::NtItem(item) => item.tokens(), - Nonterminal::NtStmt(stmt) => stmt.tokens(), - Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(), - Nonterminal::NtPat(pat) => pat.tokens(), - Nonterminal::NtTy(ty) => ty.tokens(), - Nonterminal::NtMeta(attr_item) => attr_item.tokens(), - Nonterminal::NtPath(path) => path.tokens(), - Nonterminal::NtVis(vis) => vis.tokens(), - Nonterminal::NtBlock(block) => block.tokens(), + Nonterminal::NtItem(item, _) => item.tokens(), + Nonterminal::NtStmt(stmt, _) => stmt.tokens(), + Nonterminal::NtExpr(expr, _) | Nonterminal::NtLiteral(expr, _) => expr.tokens(), + Nonterminal::NtPat(pat, _) => pat.tokens(), + Nonterminal::NtTy(ty, _) => ty.tokens(), + Nonterminal::NtMeta(attr_item, _) => attr_item.tokens(), + Nonterminal::NtPath(path, _) => path.tokens(), + Nonterminal::NtVis(vis, _) => vis.tokens(), + Nonterminal::NtBlock(block, _) => block.tokens(), Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } fn tokens_mut(&mut self) -> Option<&mut Option> { match self { - Nonterminal::NtItem(item) => item.tokens_mut(), - Nonterminal::NtStmt(stmt) => stmt.tokens_mut(), - Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(), - Nonterminal::NtPat(pat) => pat.tokens_mut(), - Nonterminal::NtTy(ty) => ty.tokens_mut(), - Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(), - Nonterminal::NtPath(path) => path.tokens_mut(), - Nonterminal::NtVis(vis) => vis.tokens_mut(), - Nonterminal::NtBlock(block) => block.tokens_mut(), + Nonterminal::NtItem(item, _) => item.tokens_mut(), + Nonterminal::NtStmt(stmt, _) => stmt.tokens_mut(), + Nonterminal::NtExpr(expr, _) | Nonterminal::NtLiteral(expr, _) => expr.tokens_mut(), + Nonterminal::NtPat(pat, _) => pat.tokens_mut(), + Nonterminal::NtTy(ty, _) => ty.tokens_mut(), + Nonterminal::NtMeta(attr_item, _) => attr_item.tokens_mut(), + Nonterminal::NtPath(path, _) => path.tokens_mut(), + Nonterminal::NtVis(vis, _) => vis.tokens_mut(), + Nonterminal::NtBlock(block, _) => block.tokens_mut(), Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index db008ea139d58..310fad6a1dfe4 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -343,8 +343,8 @@ impl MetaItem { Path { span, segments, tokens: None } } Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt { - token::Nonterminal::NtMeta(item) => return item.meta(item.path.span), - token::Nonterminal::NtPath(path) => (**path).clone(), + token::Nonterminal::NtMeta(item, _) => return item.meta(item.path.span), + token::Nonterminal::NtPath(path, _) => (**path).clone(), _ => return None, }, _ => return None, diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index ba2887146cfe4..488662b31bbff 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -793,33 +793,33 @@ pub fn visit_token(t: &mut Token, vis: &mut T) { // multiple items there.... pub fn visit_nonterminal(nt: &mut token::Nonterminal, vis: &mut T) { match nt { - token::NtItem(item) => visit_clobber(item, |item| { + token::NtItem(item, _) => visit_clobber(item, |item| { // This is probably okay, because the only visitors likely to // peek inside interpolated nodes will be renamings/markings, // which map single items to single items. vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item") }), - token::NtBlock(block) => vis.visit_block(block), - token::NtStmt(stmt) => visit_clobber(stmt, |stmt| { + token::NtBlock(block, _) => vis.visit_block(block), + token::NtStmt(stmt, _) => visit_clobber(stmt, |stmt| { // See reasoning above. stmt.map(|stmt| { vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item") }) }), - token::NtPat(pat) => vis.visit_pat(pat), - token::NtExpr(expr) => vis.visit_expr(expr), - token::NtTy(ty) => vis.visit_ty(ty), - token::NtIdent(ident, _is_raw) => vis.visit_ident(ident), - token::NtLifetime(ident) => vis.visit_ident(ident), - token::NtLiteral(expr) => vis.visit_expr(expr), - token::NtMeta(item) => { + token::NtPat(pat, _) => vis.visit_pat(pat), + token::NtExpr(expr, _) => vis.visit_expr(expr), + token::NtTy(ty, _) => vis.visit_ty(ty), + token::NtIdent(ident, _is_raw, _) => vis.visit_ident(ident), + token::NtLifetime(ident, _) => vis.visit_ident(ident), + token::NtLiteral(expr, _) => vis.visit_expr(expr), + token::NtMeta(item, _) => { let AttrItem { path, args, tokens } = item.deref_mut(); vis.visit_path(path); visit_attr_args(args, vis); visit_lazy_tts(tokens, vis); } - token::NtPath(path) => vis.visit_path(path), - token::NtVis(visib) => vis.visit_vis(visib), + token::NtPath(path, _) => vis.visit_path(path), + token::NtVis(visib, _) => vis.visit_vis(visib), } } diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index e7b1343fd30d2..49e4e402186b0 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -112,7 +112,7 @@ impl Lit { } Literal(token_lit) => Some(token_lit), Interpolated(ref nt) - if let NtExpr(expr) | NtLiteral(expr) = &**nt + if let NtExpr(expr, _) | NtLiteral(expr, _) = &**nt && let ast::ExprKind::Lit(token_lit) = expr.kind => { Some(token_lit) @@ -395,7 +395,7 @@ impl Token { /// if they keep spans or perform edition checks. pub fn uninterpolated_span(&self) -> Span { match &self.kind { - Interpolated(nt) => nt.span(), + Interpolated(nt) => nt.use_span(), _ => self.span, } } @@ -551,8 +551,8 @@ impl Token { Literal(..) | BinOp(Minus) => true, Ident(name, false) if name.is_bool_lit() => true, Interpolated(ref nt) => match &**nt { - NtLiteral(_) => true, - NtExpr(e) => match &e.kind { + NtLiteral(..) => true, + NtExpr(e, _) => match &e.kind { ast::ExprKind::Lit(_) => true, ast::ExprKind::Unary(ast::UnOp::Neg, e) => { matches!(&e.kind, ast::ExprKind::Lit(_)) @@ -572,10 +572,10 @@ impl Token { pub fn uninterpolate(&self) -> Cow<'_, Token> { match &self.kind { Interpolated(nt) => match **nt { - NtIdent(ident, is_raw) => { + NtIdent(ident, is_raw, _) => { Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) } - NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), + NtLifetime(ident, _) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), _ => Cow::Borrowed(self), }, _ => Cow::Borrowed(self), @@ -589,7 +589,7 @@ impl Token { match &self.kind { &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), Interpolated(nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident, is_raw)), + NtIdent(ident, is_raw, _) => Some((ident, is_raw)), _ => None, }, _ => None, @@ -603,7 +603,7 @@ impl Token { match &self.kind { &Lifetime(name) => Some(Ident::new(name, self.span)), Interpolated(nt) => match **nt { - NtLifetime(ident) => Some(ident), + NtLifetime(ident, _) => Some(ident), _ => None, }, _ => None, @@ -640,7 +640,7 @@ impl Token { /// (which happens while parsing the result of macro expansion)? pub fn is_whole_expr(&self) -> bool { if let Interpolated(nt) = &self.kind - && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt + && let NtExpr(..) | NtLiteral(..) | NtPath(..) | NtBlock(..) = **nt { return true; } @@ -803,19 +803,19 @@ impl PartialEq for Token { #[derive(Clone, Encodable, Decodable)] /// For interpolation during macro expansion. pub enum Nonterminal { - NtItem(P), - NtBlock(P), - NtStmt(P), - NtPat(P), - NtExpr(P), - NtTy(P), - NtIdent(Ident, /* is_raw */ bool), - NtLifetime(Ident), - NtLiteral(P), + NtItem(P, Span), + NtBlock(P, Span), + NtStmt(P, Span), + NtPat(P, Span), + NtExpr(P, Span), + NtTy(P, Span), + NtIdent(Ident, /* is_raw */ bool, Span), + NtLifetime(Ident, Span), + NtLiteral(P, Span), /// Stuff inside brackets for attributes - NtMeta(P), - NtPath(P), - NtVis(P), + NtMeta(P, Span), + NtPath(P, Span), + NtVis(P, Span), } #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)] @@ -897,35 +897,52 @@ impl fmt::Display for NonterminalKind { } impl Nonterminal { - pub fn span(&self) -> Span { + pub fn use_span(&self) -> Span { match self { - NtItem(item) => item.span, - NtBlock(block) => block.span, - NtStmt(stmt) => stmt.span, - NtPat(pat) => pat.span, - NtExpr(expr) | NtLiteral(expr) => expr.span, - NtTy(ty) => ty.span, - NtIdent(ident, _) | NtLifetime(ident) => ident.span, - NtMeta(attr_item) => attr_item.span(), - NtPath(path) => path.span, - NtVis(vis) => vis.span, + NtItem(item, _) => item.span, + NtBlock(block, _) => block.span, + NtStmt(stmt, _) => stmt.span, + NtPat(pat, _) => pat.span, + NtExpr(expr, _) | NtLiteral(expr, _) => expr.span, + NtTy(ty, _) => ty.span, + NtIdent(ident, _, _) | NtLifetime(ident, _) => ident.span, + NtMeta(attr_item, _) => attr_item.span(), + NtPath(path, _) => path.span, + NtVis(vis, _) => vis.span, + } + } + + pub fn def_span(&self) -> Span { + match self { + NtItem(_, span) + | NtBlock(_, span) + | NtStmt(_, span) + | NtPat(_, span) + | NtExpr(_, span) + | NtLiteral(_, span) + | NtTy(_, span) + | NtIdent(_, _, span) + | NtLifetime(_, span) + | NtMeta(_, span) + | NtPath(_, span) + | NtVis(_, span) => *span, } } pub fn descr(&self) -> &'static str { match self { - NtItem(_) => "item", - NtBlock(_) => "block", - NtStmt(_) => "statement", - NtPat(_) => "pattern", - NtExpr(_) => "expression", - NtLiteral(_) => "literal", - NtTy(_) => "type", + NtItem(..) => "item", + NtBlock(..) => "block", + NtStmt(..) => "statement", + NtPat(..) => "pattern", + NtExpr(..) => "expression", + NtLiteral(..) => "literal", + NtTy(..) => "type", NtIdent(..) => "identifier", - NtLifetime(_) => "lifetime", - NtMeta(_) => "attribute", - NtPath(_) => "path", - NtVis(_) => "visibility", + NtLifetime(..) => "lifetime", + NtMeta(..) => "attribute", + NtPath(..) => "path", + NtVis(..) => "visibility", } } } @@ -933,10 +950,10 @@ impl Nonterminal { impl PartialEq for Nonterminal { fn eq(&self, rhs: &Self) -> bool { match (self, rhs) { - (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { + (NtIdent(ident_lhs, is_raw_lhs, _), NtIdent(ident_rhs, is_raw_rhs, _)) => { ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs } - (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, + (NtLifetime(ident_lhs, _), NtLifetime(ident_rhs, _)) => ident_lhs == ident_rhs, // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them // correctly based on data from AST. This will prevent them from matching each other // in macros. The comparison will become possible only when each nonterminal has an @@ -982,7 +999,7 @@ mod size_asserts { // tidy-alphabetical-start static_assert_size!(Lit, 12); static_assert_size!(LitKind, 2); - static_assert_size!(Nonterminal, 16); + static_assert_size!(Nonterminal, 24); static_assert_size!(Token, 24); static_assert_size!(TokenKind, 16); // tidy-alphabetical-end diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 1e18b1232de76..f0270144b23d4 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -453,31 +453,31 @@ impl TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { match nt { - Nonterminal::NtIdent(ident, is_raw) => { + Nonterminal::NtIdent(ident, is_raw, _) => { TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span) } - Nonterminal::NtLifetime(ident) => { + Nonterminal::NtLifetime(ident, _) => { TokenStream::token_alone(token::Lifetime(ident.name), ident.span) } - Nonterminal::NtItem(item) => TokenStream::from_ast(item), - Nonterminal::NtBlock(block) => TokenStream::from_ast(block), - Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { + Nonterminal::NtItem(item, _) => TokenStream::from_ast(item), + Nonterminal::NtBlock(block, _) => TokenStream::from_ast(block), + Nonterminal::NtStmt(stmt, _) if let StmtKind::Empty = stmt.kind => { // FIXME: Properly collect tokens for empty statements. TokenStream::token_alone(token::Semi, stmt.span) } - Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt), - Nonterminal::NtPat(pat) => TokenStream::from_ast(pat), - Nonterminal::NtTy(ty) => TokenStream::from_ast(ty), - Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr), - Nonterminal::NtPath(path) => TokenStream::from_ast(path), - Nonterminal::NtVis(vis) => TokenStream::from_ast(vis), - Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr), + Nonterminal::NtStmt(stmt, _) => TokenStream::from_ast(stmt), + Nonterminal::NtPat(pat, _) => TokenStream::from_ast(pat), + Nonterminal::NtTy(ty, _) => TokenStream::from_ast(ty), + Nonterminal::NtMeta(attr, _) => TokenStream::from_ast(attr), + Nonterminal::NtPath(path, _) => TokenStream::from_ast(path), + Nonterminal::NtVis(vis, _) => TokenStream::from_ast(vis), + Nonterminal::NtExpr(expr, _) | Nonterminal::NtLiteral(expr, _) => TokenStream::from_ast(expr), } } fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { match &token.kind { - token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => { + token::Interpolated(nt) if let token::NtIdent(ident, is_raw, _) = **nt => { TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) } token::Interpolated(nt) => TokenTree::Delimited( diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 8b7e91882fcc7..650b0d3e38ed5 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -731,18 +731,18 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere fn nonterminal_to_string(&self, nt: &Nonterminal) -> String { match nt { - token::NtExpr(e) => self.expr_to_string(e), - token::NtMeta(e) => self.attr_item_to_string(e), - token::NtTy(e) => self.ty_to_string(e), - token::NtPath(e) => self.path_to_string(e), - token::NtItem(e) => self.item_to_string(e), - token::NtBlock(e) => self.block_to_string(e), - token::NtStmt(e) => self.stmt_to_string(e), - token::NtPat(e) => self.pat_to_string(e), - token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(*e, *is_raw).to_string(), - token::NtLifetime(e) => e.to_string(), - token::NtLiteral(e) => self.expr_to_string(e), - token::NtVis(e) => self.vis_to_string(e), + token::NtExpr(e, _) => self.expr_to_string(e), + token::NtMeta(e, _) => self.attr_item_to_string(e), + token::NtTy(e, _) => self.ty_to_string(e), + token::NtPath(e, _) => self.path_to_string(e), + token::NtItem(e, _) => self.item_to_string(e), + token::NtBlock(e, _) => self.block_to_string(e), + token::NtStmt(e, _) => self.stmt_to_string(e), + token::NtPat(e, _) => self.pat_to_string(e), + token::NtIdent(e, is_raw, _) => IdentPrinter::for_ast_ident(*e, *is_raw).to_string(), + token::NtLifetime(e, _) => e.to_string(), + token::NtLiteral(e, _) => self.expr_to_string(e), + token::NtVis(e, _) => self.vis_to_string(e), } } diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index c4d2a374f0c67..f9342da302028 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -1498,8 +1498,8 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &P pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &ParseSess) -> bool { let item = match nt { - Nonterminal::NtItem(item) => item, - Nonterminal::NtStmt(stmt) => match &stmt.kind { + Nonterminal::NtItem(item, _) => item, + Nonterminal::NtStmt(stmt, _) => match &stmt.kind { ast::StmtKind::Item(item) => item, _ => return false, }, diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 7e85beaadcbcc..b59c93499c4b8 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -397,7 +397,7 @@ pub(crate) enum NamedMatch { MatchedTokenTree(rustc_ast::tokenstream::TokenTree), // A metavar match of any type other than `tt`. - MatchedNonterminal(Lrc), + MatchedNonterminal(Lrc, Span), } /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) @@ -469,8 +469,10 @@ impl TtParser { let matcher_loc = &matcher[mp.idx]; track.before_match_loc(self, matcher_loc); + info!(?matcher_loc); match matcher_loc { MatcherLoc::Token { token: t } => { + info!(?t, ?token); // If it's a doc comment, we just ignore it and move on to the next tt in the // matcher. This is a bug, but #95267 showed that existing programs rely on // this behaviour, and changing it would require some care and a transition @@ -678,7 +680,7 @@ impl TtParser { { // We use the span of the metavariable declaration to determine any // edition-specific matching behavior for non-terminals. - let nt = match parser.to_mut().parse_nonterminal(kind) { + let nt = match parser.to_mut().parse_nonterminal(kind, span) { Err(mut err) => { let guarantee = err.span_label( span, @@ -692,7 +694,7 @@ impl TtParser { Ok(nt) => nt, }; let m = match nt { - ParseNtResult::Nt(nt) => MatchedNonterminal(Lrc::new(nt)), + ParseNtResult::Nt(nt) => MatchedNonterminal(Lrc::new(nt), span), ParseNtResult::Tt(tt) => MatchedTokenTree(tt), }; mp.push_match(next_metavar, seq_depth, m); @@ -755,7 +757,10 @@ impl TtParser { if let &MatcherLoc::MetaVarDecl { span, bind, kind, .. } = loc { if kind.is_some() { match ret_val.entry(MacroRulesNormalizedIdent::new(bind)) { - Vacant(spot) => spot.insert(res.next().unwrap()), + Vacant(spot) => { + info!(?spot, ?span, ?kind, ?bind); + spot.insert(res.next().unwrap()) + } Occupied(..) => { return Error(span, format!("duplicated bind name: {bind}")); } diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 15e7ab3fe3ee6..259a2c9b52667 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -215,18 +215,33 @@ pub(super) fn transcribe<'a>( // Find the matched nonterminal from the macro invocation, and use it to replace // the meta-var. let ident = MacroRulesNormalizedIdent::new(original_ident); + let x = original_ident.normalize_to_macro_rules().span; + info!(?interp, ?x, ?original_ident.span); if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + info!(?cur_matched); match cur_matched { MatchedTokenTree(tt) => { // `tt`s are emitted into the output stream directly as "raw tokens", // without wrapping them into groups. result.push(tt.clone()); } - MatchedNonterminal(nt) => { + MatchedNonterminal(nt, span) => { + info!(?span, ?nt, "nt, span: {:?} {:?}", nt.use_span(), nt.def_span()); // Other variables are emitted into the output stream as groups with // `Delimiter::Invisible` to maintain parsing priorities. // `Interpolated` is currently used for such groups in rustc parser. marker.visit_span(&mut sp); + // let mut data = span.expn_data(); + // let mut span = *span; + // marker.visit_span(&mut span); + // data.call_site = span; + // // let mut span = *span; + // // span = span.apply_mark(marker.0.to_expn_id(), marker.1); + // // marker.visit_span(&mut span); + // let expn_id = LocalExpnId::fresh( + // data, + // self.create_stable_hashing_context(), + // ); result .push(TokenTree::token_alone(token::Interpolated(nt.clone()), sp)); } @@ -298,7 +313,7 @@ fn lookup_cur_matched<'a>( interpolations.get(&ident).map(|mut matched| { for &(idx, _) in repeats { match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => break, + MatchedTokenTree(_) | MatchedNonterminal(_, _) => break, MatchedSeq(ads) => matched = ads.get(idx).unwrap(), } } @@ -388,7 +403,9 @@ fn lockstep_iter_size( let name = MacroRulesNormalizedIdent::new(*name); match lookup_cur_matched(name, interpolations, repeats) { Some(matched) => match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained, + MatchedTokenTree(_) | MatchedNonterminal(_, _) => { + LockstepIterSize::Unconstrained + } MatchedSeq(ads) => LockstepIterSize::Constraint(ads.len(), name), }, _ => LockstepIterSize::Unconstrained, @@ -437,7 +454,7 @@ fn count_repetitions<'a>( sp: &DelimSpan, ) -> PResult<'a, usize> { match matched { - MatchedTokenTree(_) | MatchedNonterminal(_) => { + MatchedTokenTree(_) | MatchedNonterminal(_, _) => { if declared_lhs_depth == 0 { return Err(cx.create_err(CountRepetitionMisplaced { span: sp.entire() })); } diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index c617cd76e3cb0..75166d101939c 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -122,8 +122,8 @@ impl MultiItemModifier for DeriveProcMacro { let hack = crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.parse_sess); let input = if hack { let nt = match item { - Annotatable::Item(item) => token::NtItem(item), - Annotatable::Stmt(stmt) => token::NtStmt(stmt), + Annotatable::Item(item) => token::NtItem(item, span), + Annotatable::Stmt(stmt) => token::NtStmt(stmt, span), _ => unreachable!(), }; TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP) diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 2dc9b51a37ea0..f5a4df590cb8c 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -226,7 +226,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + Interpolated(nt) if let NtIdent(ident, is_raw, _) = *nt => { trees.push(TokenTree::Ident(Ident { sym: ident.name, is_raw, span: ident.span })) } diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 104de47b97dd1..73ad40919f3bb 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -250,7 +250,7 @@ impl<'a> Parser<'a> { pub fn parse_attr_item(&mut self, capture_tokens: bool) -> PResult<'a, ast::AttrItem> { let item = match &self.token.kind { token::Interpolated(nt) => match &**nt { - Nonterminal::NtMeta(item) => Some(item.clone().into_inner()), + Nonterminal::NtMeta(item, _) => Some(item.clone().into_inner()), _ => None, }, _ => None, @@ -370,7 +370,7 @@ impl<'a> Parser<'a> { pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> { let nt_meta = match &self.token.kind { token::Interpolated(nt) => match &**nt { - token::NtMeta(e) => Some(e.clone()), + token::NtMeta(e, _) => Some(e.clone()), _ => None, }, _ => None, diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 792297b6ea21c..61daf0124f739 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -2172,10 +2172,8 @@ impl<'a> Parser<'a> { let mut tok = self.token.clone(); let mut labels = vec![]; while let TokenKind::Interpolated(node) = &tok.kind { - // FIXME: figure out how to point at `$x:pat` when `tok` - // is a `NtPat` corresponding to `$x`. let tokens = node.tokens(); - labels.push((node.span(), node.descr())); + labels.push(node.clone()); if let Some(tokens) = tokens && let tokens = tokens.to_attr_token_stream() && let tokens = tokens.0.deref() @@ -2188,23 +2186,31 @@ impl<'a> Parser<'a> { } let mut iter = labels.into_iter().peekable(); let mut show_link = false; - while let Some((span, descr)) = iter.next() { - if let Some((next_span, next_descr)) = iter.peek() - && next_descr != &descr - { - err.span_label( - *next_span, - format!("this is interpreted as {next_descr}"), - ); - err.span_label( - span, - format!( - "this is interpreted as {}, but it was previously interpreted as {}", - descr, - next_descr, - ), - ); - show_link = true; + while let Some(node) = iter.next() { + let descr = node.descr(); + if let Some(next) = iter.peek() { + let next_descr = next.descr(); + if next_descr != descr { + err.span_label( + next.def_span(), + format!("this macro fragment matcher is {next_descr}"), + ); + err.span_label( + node.def_span(), + format!("this macro fragment matcher is {descr}"), + ); + err.span_label(next.use_span(), format!("this is interpreted as {next_descr}")); + err.span_label( + node.use_span(), + format!( + "this is interpreted as {}, but it was previously interpreted as {}", + descr, next_descr, + ), + ); + show_link = true; + } else { + err.span_label(node.def_span(), ""); + } } } if show_link { diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 91bb2d9eb666f..572bf3878e1cc 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -47,17 +47,17 @@ macro_rules! maybe_whole_expr { ($p:expr) => { if let token::Interpolated(nt) = &$p.token.kind { match &**nt { - token::NtExpr(e) | token::NtLiteral(e) => { + token::NtExpr(e, _) | token::NtLiteral(e, _) => { let e = e.clone(); $p.bump(); return Ok(e); } - token::NtPath(path) => { + token::NtPath(path, _) => { let path = (**path).clone(); $p.bump(); return Ok($p.mk_expr($p.prev_token.span, ExprKind::Path(None, path))); } - token::NtBlock(block) => { + token::NtBlock(block, _) => { let block = block.clone(); $p.bump(); return Ok($p.mk_expr($p.prev_token.span, ExprKind::Block(block, None))); @@ -1946,7 +1946,7 @@ impl<'a> Parser<'a> { mk_lit_char: impl FnOnce(Symbol, Span) -> L, ) -> PResult<'a, L> { if let token::Interpolated(nt) = &self.token.kind - && let token::NtExpr(e) | token::NtLiteral(e) = &**nt + && let token::NtExpr(e, _) | token::NtLiteral(e, _) = &**nt && matches!(e.kind, ExprKind::Err) { let mut err = errors::InvalidInterpolatedExpression { span: self.token.span } diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index cc54cc5bebbdd..5f40d575c4ac1 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -122,7 +122,7 @@ impl<'a> Parser<'a> { ) -> PResult<'a, Option> { // Don't use `maybe_whole` so that we have precise control // over when we bump the parser - if let token::Interpolated(nt) = &self.token.kind && let token::NtItem(item) = &**nt { + if let token::Interpolated(nt) = &self.token.kind && let token::NtItem(item, _) = &**nt { let mut item = item.clone(); self.bump(); diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 25a13c5cdaa4c..883964b5e4080 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -92,7 +92,7 @@ pub enum TrailingToken { macro_rules! maybe_whole { ($p:expr, $constructor:ident, |$x:ident| $e:expr) => { if let token::Interpolated(nt) = &$p.token.kind { - if let token::$constructor(x) = &**nt { + if let token::$constructor(x, _) = &**nt { let $x = x.clone(); $p.bump(); return Ok($e); @@ -109,7 +109,7 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath { && $self.may_recover() && $self.look_ahead(1, |t| t == &token::ModSep) && let token::Interpolated(nt) = &$self.token.kind - && let token::NtTy(ty) = &**nt + && let token::NtTy(ty, _) = &**nt { let ty = ty.clone(); $self.bump(); diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index ff059a7e865a4..be1509fdf7e1e 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -5,6 +5,7 @@ use rustc_ast_pretty::pprust; use rustc_errors::IntoDiagnostic; use rustc_errors::PResult; use rustc_span::symbol::{kw, Ident}; +use rustc_span::Span; use crate::errors::UnexpectedNonterminal; use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma}; @@ -21,19 +22,19 @@ impl<'a> Parser<'a> { /// Checks whether the non-terminal may contain a single (non-keyword) identifier. fn may_be_ident(nt: &token::Nonterminal) -> bool { match nt { - NtStmt(_) - | NtPat(_) - | NtExpr(_) - | NtTy(_) + NtStmt(..) + | NtPat(..) + | NtExpr(..) + | NtTy(..) | NtIdent(..) - | NtLiteral(_) // `true`, `false` - | NtMeta(_) - | NtPath(_) => true, + | NtLiteral(..) // `true`, `false` + | NtMeta(..) + | NtPath(..) => true, - NtItem(_) - | NtBlock(_) - | NtVis(_) - | NtLifetime(_) => false, + NtItem(..) + | NtBlock(..) + | NtVis(..) + | NtLifetime(..) => false, } } @@ -56,9 +57,9 @@ impl<'a> Parser<'a> { NonterminalKind::Block => match &token.kind { token::OpenDelim(Delimiter::Brace) => true, token::Interpolated(nt) => match **nt { - NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, - NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_) - | NtVis(_) => false, + NtBlock(..) | NtLifetime(..) | NtStmt(..) | NtExpr(..) | NtLiteral(..) => true, + NtItem(..) | NtPat(..) | NtTy(..) | NtIdent(..) | NtMeta(..) | NtPath(..) + | NtVis(..) => false, }, _ => false, }, @@ -90,7 +91,7 @@ impl<'a> Parser<'a> { NonterminalKind::Lifetime => match &token.kind { token::Lifetime(_) => true, token::Interpolated(nt) => { - matches!(**nt, NtLifetime(_)) + matches!(**nt, NtLifetime(..)) } _ => false, }, @@ -103,7 +104,11 @@ impl<'a> Parser<'a> { /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`). Inlined because there is only one call /// site. #[inline] - pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, ParseNtResult> { + pub fn parse_nonterminal( + &mut self, + kind: NonterminalKind, + span: Span, + ) -> PResult<'a, ParseNtResult> { // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro, // which requires having captured tokens available. Since we cannot determine // in advance whether or not a proc-macro will be (transitively) invoked, @@ -112,7 +117,7 @@ impl<'a> Parser<'a> { // Note that TT is treated differently to all the others. NonterminalKind::TT => return Ok(ParseNtResult::Tt(self.parse_token_tree())), NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? { - Some(item) => NtItem(item), + Some(item) => NtItem(item, span), None => { return Err(UnexpectedNonterminal::Item(self.token.span) .into_diagnostic(&self.sess.span_diagnostic)); @@ -121,10 +126,10 @@ impl<'a> Parser<'a> { NonterminalKind::Block => { // While a block *expression* may have attributes (e.g. `#[my_attr] { ... }`), // the ':block' matcher does not support them - NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?) + NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?, span) } NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? { - Some(s) => NtStmt(P(s)), + Some(s) => NtStmt(P(s), span), None => { return Err(UnexpectedNonterminal::Statement(self.token.span) .into_diagnostic(&self.sess.span_diagnostic)); @@ -140,19 +145,21 @@ impl<'a> Parser<'a> { CommaRecoveryMode::EitherTupleOrPipe, ), _ => unreachable!(), - })?) + })?, span) } - NonterminalKind::Expr => NtExpr(self.parse_expr_force_collect()?), + NonterminalKind::Expr => NtExpr(self.parse_expr_force_collect()?, span), NonterminalKind::Literal => { // The `:literal` matcher does not support attributes NtLiteral( self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?, + span, ) } NonterminalKind::Ty => NtTy( self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?, + span, ), // this could be handled like a token, since it is one @@ -160,7 +167,7 @@ impl<'a> Parser<'a> { if let Some((ident, is_raw)) = get_macro_ident(&self.token) => { self.bump(); - NtIdent(ident, is_raw) + NtIdent(ident, is_raw, span) } NonterminalKind::Ident => { return Err(UnexpectedNonterminal::Ident { @@ -170,14 +177,16 @@ impl<'a> Parser<'a> { } NonterminalKind::Path => NtPath( P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?), + span, ), - NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(true)?)), + NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(true)?), span), NonterminalKind::Vis => NtVis( P(self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?), + span, ), NonterminalKind::Lifetime => { if self.check_lifetime() { - NtLifetime(self.expect_lifetime().ident) + NtLifetime(self.expect_lifetime().ident, span) } else { return Err(UnexpectedNonterminal::Lifetime { span: self.token.span, @@ -192,7 +201,7 @@ impl<'a> Parser<'a> { panic!( "Missing tokens for nt {:?} at {:?}: {:?}", nt, - nt.span(), + nt.use_span(), pprust::nonterminal_to_string(&nt) ); } diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs index 3e4e927891072..170eda50b7e18 100644 --- a/compiler/rustc_parse/src/parser/pat.rs +++ b/compiler/rustc_parse/src/parser/pat.rs @@ -591,7 +591,7 @@ impl<'a> Parser<'a> { // Make sure we don't allow e.g. `let mut $p;` where `$p:pat`. if let token::Interpolated(nt) = &self.token.kind { - if let token::NtPat(_) = **nt { + if let token::NtPat(..) = **nt { self.expected_ident_found_err().emit(); } } diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 445516c03a15e..7861dbd54c157 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -185,7 +185,7 @@ impl<'a> Parser<'a> { }); if let token::Interpolated(nt) = &self.token.kind { - if let token::NtTy(ty) = &**nt { + if let token::NtTy(ty, _) = &**nt { if let ast::TyKind::Path(None, path) = &ty.kind { let path = path.clone(); self.bump(); diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 12c267351b9aa..b990c06dd9611 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -52,7 +52,7 @@ impl<'a> Parser<'a> { // Don't use `maybe_whole` so that we have precise control // over when we bump the parser - if let token::Interpolated(nt) = &self.token.kind && let token::NtStmt(stmt) = &**nt { + if let token::Interpolated(nt) = &self.token.kind && let token::NtStmt(stmt, _) = &**nt { let mut stmt = stmt.clone(); self.bump(); stmt.visit_attrs(|stmt_attrs| { diff --git a/tests/ui/macros/trace_faulty_macros.stderr b/tests/ui/macros/trace_faulty_macros.stderr index c740d5d09fa86..ae21aa17b8115 100644 --- a/tests/ui/macros/trace_faulty_macros.stderr +++ b/tests/ui/macros/trace_faulty_macros.stderr @@ -73,10 +73,14 @@ error: expected expression, found pattern `1 + 1` --> $DIR/trace_faulty_macros.rs:49:37 | LL | (let $p:pat = $e:expr) => {test!(($p,$e))}; - | -- this is interpreted as pattern, but it was previously interpreted as expression + | ------- -- this is interpreted as pattern, but it was previously interpreted as expression + | | + | this macro fragment matcher is expression ... LL | (($p:pat, $e:pat)) => {let $p = $e;}; - | ^^ expected expression + | ------ ^^ expected expression + | | + | this macro fragment matcher is pattern ... LL | test!(let x = 1+1); | ------------------