Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rework pattern and expression nonterminal kinds. #126851

Merged
merged 3 commits into from
Jun 23, 2024
Merged

Rework pattern and expression nonterminal kinds. #126851

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
470b0e9
Import `NonterminalKind` in `compiler/rustc_expand/src/mbe/quoted.rs`.
nnethercote Jun 22, 2024
70fa67c
Tweak some ugly formatting.
nnethercote Jun 22, 2024
e2aa38e
Rework pattern and expression nonterminal kinds.
nnethercote Jun 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 44 additions & 29 deletions compiler/rustc_ast/src/token.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
pub use BinOpToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use NtExprKind::*;
pub use NtPatKind::*;
pub use TokenKind::*;

use crate::ast;
Expand Down Expand Up @@ -871,6 +873,27 @@ impl PartialEq<TokenKind> for Token {
}
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
pub enum NtPatKind {
// Matches or-patterns. Was written using `pat` in edition 2021 or later.
PatWithOr,
// Doesn't match or-patterns.
// - `inferred`: was written using `pat` in edition 2015 or 2018.
// - `!inferred`: was written using `pat_param`.
PatParam { inferred: bool },
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)]
pub enum NtExprKind {
// Matches expressions using the post-edition 2024. Was written using
// `expr` in edition 2024 or later.
Expr,
// Matches expressions using the pre-edition 2024 rules.
// - `inferred`: was written using `expr` in edition 2021 or earlier.
// - `!inferred`: was written using `expr_2021`.
Expr2021 { inferred: bool },
}

#[derive(Clone, Encodable, Decodable)]
/// For interpolation during macro expansion.
pub enum Nonterminal {
Expand All @@ -892,19 +915,8 @@ pub enum NonterminalKind {
Item,
Block,
Stmt,
PatParam {
/// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the
/// edition of the span. This is used for diagnostics.
inferred: bool,
},
PatWithOr,
Expr,
/// Matches an expression using the rules from edition 2021 and earlier.
Expr2021 {
/// Keep track of whether the user used `:expr` or `:expr_2021` and we inferred it from the
/// edition of the span. This is used for diagnostics AND feature gating.
inferred: bool,
},
Pat(NtPatKind),
Expr(NtExprKind),
Ty,
Ident,
Lifetime,
Expand All @@ -926,20 +938,22 @@ impl NonterminalKind {
sym::item => NonterminalKind::Item,
sym::block => NonterminalKind::Block,
sym::stmt => NonterminalKind::Stmt,
sym::pat => match edition() {
Edition::Edition2015 | Edition::Edition2018 => {
NonterminalKind::PatParam { inferred: true }
sym::pat => {
if edition().at_least_rust_2021() {
NonterminalKind::Pat(PatWithOr)
} else {
NonterminalKind::Pat(PatParam { inferred: true })
}
Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr,
},
sym::pat_param => NonterminalKind::PatParam { inferred: false },
sym::expr => match edition() {
Edition::Edition2015 | Edition::Edition2018 | Edition::Edition2021 => {
NonterminalKind::Expr2021 { inferred: true }
}
sym::pat_param => NonterminalKind::Pat(PatParam { inferred: false }),
sym::expr => {
if edition().at_least_rust_2024() {
NonterminalKind::Expr(Expr)
} else {
NonterminalKind::Expr(Expr2021 { inferred: true })
}
Edition::Edition2024 => NonterminalKind::Expr,
},
sym::expr_2021 => NonterminalKind::Expr2021 { inferred: false },
}
sym::expr_2021 => NonterminalKind::Expr(Expr2021 { inferred: false }),
sym::ty => NonterminalKind::Ty,
sym::ident => NonterminalKind::Ident,
sym::lifetime => NonterminalKind::Lifetime,
Expand All @@ -951,15 +965,16 @@ impl NonterminalKind {
_ => return None,
})
}

fn symbol(self) -> Symbol {
match self {
NonterminalKind::Item => sym::item,
NonterminalKind::Block => sym::block,
NonterminalKind::Stmt => sym::stmt,
NonterminalKind::PatParam { inferred: false } => sym::pat_param,
NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: true } => sym::expr,
NonterminalKind::Expr2021 { inferred: false } => sym::expr_2021,
NonterminalKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
NonterminalKind::Pat(PatParam { inferred: false }) => sym::pat_param,
NonterminalKind::Expr(Expr2021 { inferred: true } | Expr) => sym::expr,
NonterminalKind::Expr(Expr2021 { inferred: false }) => sym::expr_2021,
NonterminalKind::Ty => sym::ty,
NonterminalKind::Ident => sym::ident,
NonterminalKind::Lifetime => sym::lifetime,
Expand Down
25 changes: 14 additions & 11 deletions compiler/rustc_expand/src/mbe/macro_rules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ use crate::mbe::transcribe::transcribe;

use ast::token::IdentIsRaw;
use rustc_ast as ast;
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
use rustc_ast::token::{
self, Delimiter, NonterminalKind, NtPatKind::*, Token, TokenKind, TokenKind::*,
};
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::{NodeId, DUMMY_NODE_ID};
use rustc_ast_pretty::pprust;
Expand Down Expand Up @@ -1145,14 +1147,17 @@ fn check_matcher_core<'tt>(
// Macros defined in the current crate have a real node id,
// whereas macros from an external crate have a dummy id.
if def.id != DUMMY_NODE_ID
&& matches!(kind, NonterminalKind::PatParam { inferred: true })
&& matches!(next_token, TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or))
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
&& matches!(
next_token,
TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or)
)
{
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
name,
Some(NonterminalKind::PatParam { inferred: false }),
Some(NonterminalKind::Pat(PatParam { inferred: false })),
));
sess.psess.buffer_lint(
RUST_2021_INCOMPATIBLE_OR_PATTERNS,
Expand Down Expand Up @@ -1185,14 +1190,14 @@ fn check_matcher_core<'tt>(
);
err.span_label(sp, format!("not allowed after `{kind}` fragments"));

if kind == NonterminalKind::PatWithOr
if kind == NonterminalKind::Pat(PatWithOr)
&& sess.psess.edition.at_least_rust_2021()
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
{
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
name,
Some(NonterminalKind::PatParam { inferred: false }),
Some(NonterminalKind::Pat(PatParam { inferred: false })),
));
err.span_suggestion(
span,
Expand Down Expand Up @@ -1292,9 +1297,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
// maintain
IsInFollow::Yes
}
NonterminalKind::Stmt
| NonterminalKind::Expr
| NonterminalKind::Expr2021 { inferred: _ } => {
NonterminalKind::Stmt | NonterminalKind::Expr(_) => {
const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"];
match tok {
TokenTree::Token(token) => match token.kind {
Expand All @@ -1304,7 +1307,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
_ => IsInFollow::No(TOKENS),
}
}
NonterminalKind::PatParam { .. } => {
NonterminalKind::Pat(PatParam { .. }) => {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
Expand All @@ -1317,7 +1320,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
_ => IsInFollow::No(TOKENS),
}
}
NonterminalKind::PatWithOr => {
NonterminalKind::Pat(PatWithOr) => {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
Expand Down
55 changes: 25 additions & 30 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use crate::errors;
use crate::mbe::macro_parser::count_metavar_decls;
use crate::mbe::{Delimited, KleeneOp, KleeneToken, MetaVarExpr, SequenceRepetition, TokenTree};

use rustc_ast::token::{self, Delimiter, IdentIsRaw, Token};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, NtExprKind::*, Token};
use rustc_ast::{tokenstream, NodeId};
use rustc_ast_pretty::pprust;
use rustc_feature::Features;
Expand Down Expand Up @@ -85,36 +85,31 @@ pub(super) fn parse(
span.edition()
}
};
let kind =
token::NonterminalKind::from_symbol(fragment.name, edition)
.unwrap_or_else(|| {
let help = match fragment.name {
sym::expr_2021 => {
format!(
"fragment specifier `expr_2021` \
requires Rust 2021 or later\n\
{VALID_FRAGMENT_NAMES_MSG}"
)
}
_ if edition().at_least_rust_2021()
&& features
.expr_fragment_specifier_2024 =>
{
VALID_FRAGMENT_NAMES_MSG_2021.into()
}
_ => VALID_FRAGMENT_NAMES_MSG.into(),
};
sess.dcx().emit_err(
errors::InvalidFragmentSpecifier {
span,
fragment,
help,
},
);
token::NonterminalKind::Ident
let kind = NonterminalKind::from_symbol(fragment.name, edition)
.unwrap_or_else(|| {
let help = match fragment.name {
sym::expr_2021 => {
format!(
"fragment specifier `expr_2021` \
requires Rust 2021 or later\n\
{VALID_FRAGMENT_NAMES_MSG}"
)
}
_ if edition().at_least_rust_2021()
&& features.expr_fragment_specifier_2024 =>
{
VALID_FRAGMENT_NAMES_MSG_2021.into()
}
_ => VALID_FRAGMENT_NAMES_MSG.into(),
};
sess.dcx().emit_err(errors::InvalidFragmentSpecifier {
span,
fragment,
help,
});
if kind
== (token::NonterminalKind::Expr2021 { inferred: false })
NonterminalKind::Ident
});
if kind == NonterminalKind::Expr(Expr2021 { inferred: false })
&& !features.expr_fragment_specifier_2024
{
rustc_session::parse::feature_err(
Expand Down
28 changes: 12 additions & 16 deletions compiler/rustc_parse/src/parser/nonterminal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, Nonterminal::*, NonterminalKind, Token};
use rustc_ast::token::{
self, Delimiter, Nonterminal::*, NonterminalKind, NtExprKind::*, NtPatKind::*, Token,
};
use rustc_ast::HasTokens;
use rustc_ast_pretty::pprust;
use rustc_data_structures::sync::Lrc;
Expand Down Expand Up @@ -36,14 +38,14 @@ impl<'a> Parser<'a> {
}

match kind {
NonterminalKind::Expr2021 { inferred: _ } => {
NonterminalKind::Expr(Expr2021 { .. }) => {
token.can_begin_expr()
// This exception is here for backwards compatibility.
&& !token.is_keyword(kw::Let)
// This exception is here for backwards compatibility.
&& !token.is_keyword(kw::Const)
}
NonterminalKind::Expr => {
NonterminalKind::Expr(Expr) => {
token.can_begin_expr()
// This exception is here for backwards compatibility.
&& !token.is_keyword(kw::Let)
Expand Down Expand Up @@ -74,7 +76,7 @@ impl<'a> Parser<'a> {
token::Interpolated(nt) => may_be_ident(nt),
_ => false,
},
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
NonterminalKind::Pat(pat_kind) => match &token.kind {
// box, ref, mut, and other identifiers (can stricten)
token::Ident(..) | token::NtIdent(..) |
token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern
Expand All @@ -89,7 +91,7 @@ impl<'a> Parser<'a> {
token::Lt | // path (UFCS constant)
token::BinOp(token::Shl) => true, // path (double UFCS)
// leading vert `|` or-pattern
token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr),
token::BinOp(token::Or) => matches!(pat_kind, PatWithOr),
token::Interpolated(nt) => may_be_ident(nt),
_ => false,
},
Expand Down Expand Up @@ -135,31 +137,25 @@ impl<'a> Parser<'a> {
.create_err(UnexpectedNonterminal::Statement(self.token.span)));
}
},
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => {
NtPat(self.collect_tokens_no_attrs(|this| match kind {
NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None, None),
NonterminalKind::PatWithOr => this.parse_pat_allow_top_alt(
NonterminalKind::Pat(pat_kind) => {
NtPat(self.collect_tokens_no_attrs(|this| match pat_kind {
PatParam { .. } => this.parse_pat_no_top_alt(None, None),
PatWithOr => this.parse_pat_allow_top_alt(
None,
RecoverComma::No,
RecoverColon::No,
CommaRecoveryMode::EitherTupleOrPipe,
),
_ => unreachable!(),
})?)
}

NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: _ } => {
NtExpr(self.parse_expr_force_collect()?)
}
NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?),
NonterminalKind::Literal => {
// The `:literal` matcher does not support attributes
NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
}

NonterminalKind::Ty => {
NtTy(self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?)
}

// this could be handled like a token, since it is one
NonterminalKind::Ident => {
return if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
Expand Down
6 changes: 3 additions & 3 deletions src/tools/rustfmt/src/parse/macros/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use rustc_ast::token::{Delimiter, NonterminalKind, TokenKind};
use rustc_ast::token::{Delimiter, NonterminalKind, NtExprKind::*, NtPatKind::*, TokenKind};
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::{ast, ptr};
use rustc_parse::parser::{ForceCollect, Parser, Recovery};
Expand Down Expand Up @@ -48,7 +48,7 @@ fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option<MacroArg> {

parse_macro_arg!(
Expr,
NonterminalKind::Expr,
NonterminalKind::Expr(Expr),
|parser: &mut Parser<'b>| parser.parse_expr(),
|x: ptr::P<ast::Expr>| Some(x)
);
Expand All @@ -60,7 +60,7 @@ fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option<MacroArg> {
);
parse_macro_arg!(
Pat,
NonterminalKind::PatParam { inferred: false },
NonterminalKind::Pat(PatParam { inferred: false }),
|parser: &mut Parser<'b>| parser.parse_pat_no_top_alt(None, None),
|x: ptr::P<ast::Pat>| Some(x)
);
Expand Down
Loading