From 51b9dc3fba5d2afe8fc112c11bb6e7307e50af54 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Sat, 18 May 2019 08:23:52 +0200 Subject: [PATCH 1/2] Support ? Kleene operator in 2015. --- src/libsyntax/ext/tt/quoted.rs | 171 +-------------------------------- 1 file changed, 4 insertions(+), 167 deletions(-) diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index ed8395f11ad50..0e3894b5943ef 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -1,5 +1,4 @@ use crate::ast::NodeId; -use crate::early_buffered_lints::BufferedEarlyLintId; use crate::ext::tt::macro_parser; use crate::feature_gate::Features; use crate::parse::{token, ParseSess}; @@ -291,16 +290,7 @@ where macro_node_id, ); // Get the Kleene operator and optional separator - let (separator, op) = - parse_sep_and_kleene_op( - trees, - span.entire(), - sess, - features, - attrs, - edition, - macro_node_id, - ); + let (separator, op) = parse_sep_and_kleene_op(trees, span.entire(), sess); // Count the number of captured "names" (i.e., named metavars) let name_captures = macro_parser::count_names(&sequence); TokenTree::Sequence( @@ -411,164 +401,11 @@ where /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// error with the appropriate span is emitted to `sess` and a dummy value is returned. -/// -/// N.B., in the 2015 edition, `*` and `+` are the only Kleene operators, and `?` is a separator. -/// In the 2018 edition however, `?` is a Kleene operator, and not a separator. -fn parse_sep_and_kleene_op( - input: &mut Peekable, - span: Span, - sess: &ParseSess, - features: &Features, - attrs: &[ast::Attribute], - edition: Edition, - macro_node_id: NodeId, -) -> (Option, KleeneOp) -where - I: Iterator, -{ - match edition { - Edition::Edition2015 => parse_sep_and_kleene_op_2015( - input, - span, - sess, - features, - attrs, - macro_node_id, - ), - Edition::Edition2018 => parse_sep_and_kleene_op_2018(input, span, sess, features, attrs), - } -} - -// `?` is a separator (with a migration warning) and never a KleeneOp. -fn parse_sep_and_kleene_op_2015( - input: &mut Peekable, - span: Span, - sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], - macro_node_id: NodeId, -) -> (Option, KleeneOp) -where - I: Iterator, -{ - // We basically look at two token trees here, denoted as #1 and #2 below - let span = match parse_kleene_op(input, span) { - // #1 is a `+` or `*` KleeneOp - // - // `?` is ambiguous: it could be a separator (warning) or a Kleene::ZeroOrOne (error), so - // we need to look ahead one more token to be sure. - Ok(Ok((op, _))) if op != KleeneOp::ZeroOrOne => return (None, op), - - // #1 is `?` token, but it could be a Kleene::ZeroOrOne (error in 2015) without a separator - // or it could be a `?` separator followed by any Kleene operator. We need to look ahead 1 - // token to find out which. - Ok(Ok((op, op1_span))) => { - assert_eq!(op, KleeneOp::ZeroOrOne); - - // Lookahead at #2. If it is a KleenOp, then #1 is a separator. - let is_1_sep = if let Some(&tokenstream::TokenTree::Token(_, ref tok2)) = input.peek() { - kleene_op(tok2).is_some() - } else { - false - }; - - if is_1_sep { - // #1 is a separator and #2 should be a KleepeOp. - // (N.B. We need to advance the input iterator.) - match parse_kleene_op(input, span) { - // #2 is `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition. - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a Kleene op, which is the only valid option - Ok(Ok((op, _))) => { - // Warn that `?` as a separator will be deprecated - sess.buffer_lint( - BufferedEarlyLintId::QuestionMarkMacroSep, - op1_span, - macro_node_id, - "using `?` as a separator is deprecated and will be \ - a hard error in an upcoming edition", - ); - - return (Some(token::Question), op); - } - - // #2 is a random token (this is an error) :( - Ok(Err((_, _))) => op1_span, - - // #2 is not even a token at all :( - Err(_) => op1_span, - } - } else { - // `?` is not allowed as a Kleene op in 2015, - // but is allowed in the 2018 edition - sess.span_diagnostic - .struct_span_err(op1_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - } - - // #1 is a separator followed by #2, a KleeneOp - Ok(Err((tok, span))) => match parse_kleene_op(input, span) { - // #2 is a `?`, which is not allowed as a Kleene op in 2015 edition, - // but is allowed in the 2018 edition - Ok(Ok((op, op2_span))) if op == KleeneOp::ZeroOrOne => { - sess.span_diagnostic - .struct_span_err(op2_span, "expected `*` or `+`") - .note("`?` is not a macro repetition operator in the 2015 edition, \ - but is accepted in the 2018 edition") - .emit(); - - // Return a dummy - return (None, KleeneOp::ZeroOrMore); - } - - // #2 is a KleeneOp :D - Ok(Ok((op, _))) => return (Some(tok), op), - - // #2 is a random token :( - Ok(Err((_, span))) => span, - - // #2 is not a token at all :( - Err(span) => span, - }, - - // #1 is not a token - Err(span) => span, - }; - - sess.span_diagnostic.span_err(span, "expected `*` or `+`"); - - // Return a dummy - (None, KleeneOp::ZeroOrMore) -} - -// `?` is a Kleene op, not a separator -fn parse_sep_and_kleene_op_2018( - input: &mut Peekable, +fn parse_sep_and_kleene_op( + input: &mut Peekable>, span: Span, sess: &ParseSess, - _features: &Features, - _attrs: &[ast::Attribute], -) -> (Option, KleeneOp) -where - I: Iterator, -{ +) -> (Option, KleeneOp) { // We basically look at two token trees here, denoted as #1 and #2 below let span = match parse_kleene_op(input, span) { // #1 is a `?` (needs feature gate) From 695b601b7c85fe146f0837953826b29f3877a80d Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Sat, 18 May 2019 08:24:56 +0200 Subject: [PATCH 2/2] Update tests since ? macro op is supported on 2015. --- .../macros/macro-at-most-once-rep-2015.rs | 33 ++++++ ...-rep.rs => macro-at-most-once-rep-2018.rs} | 0 src/test/ui/issues/issue-39388.rs | 2 +- src/test/ui/issues/issue-39388.stderr | 2 +- .../macro-at-most-once-rep-2015-ques-rep.rs | 13 --- ...acro-at-most-once-rep-2015-ques-rep.stderr | 18 --- .../macro-at-most-once-rep-2015-ques-sep.rs | 28 ----- ...acro-at-most-once-rep-2015-ques-sep.stderr | 24 ---- .../ui/macros/macro-at-most-once-rep-2015.rs | 41 +++++++ .../macros/macro-at-most-once-rep-2015.stderr | 107 ++++++++++++++++++ src/test/ui/parser/macro/issue-33569.rs | 2 +- src/test/ui/parser/macro/issue-33569.stderr | 2 +- 12 files changed, 185 insertions(+), 87 deletions(-) create mode 100644 src/test/run-pass/macros/macro-at-most-once-rep-2015.rs rename src/test/run-pass/macros/{macro-at-most-once-rep.rs => macro-at-most-once-rep-2018.rs} (100%) delete mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.rs delete mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.stderr delete mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.rs delete mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.stderr create mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015.rs create mode 100644 src/test/ui/macros/macro-at-most-once-rep-2015.stderr diff --git a/src/test/run-pass/macros/macro-at-most-once-rep-2015.rs b/src/test/run-pass/macros/macro-at-most-once-rep-2015.rs new file mode 100644 index 0000000000000..72dfc119966dc --- /dev/null +++ b/src/test/run-pass/macros/macro-at-most-once-rep-2015.rs @@ -0,0 +1,33 @@ +// run-pass +#![allow(unused_mut)] +// The logic for parsing Kleene operators in macros has a special case to disambiguate `?`. +// Specifically, `$(pat)?` is the ZeroOrOne operator whereas `$(pat)?+` or `$(pat)?*` are the +// ZeroOrMore and OneOrMore operators using `?` as a separator. These tests are intended to +// exercise that logic in the macro parser. +// +// Moreover, we also throw in some tests for using a separator with `?`, which is meaningless but +// included for consistency with `+` and `*`. +// +// This test focuses on non-error cases and making sure the correct number of repetitions happen. + +// edition:2015 + +macro_rules! foo { + ($($a:ident)? ; $num:expr) => { { + let mut x = 0; + + $( + x += $a; + )? + + assert_eq!(x, $num); + } } +} + +pub fn main() { + let a = 1; + + // accept 0 or 1 repetitions + foo!( ; 0); + foo!(a ; 1); +} diff --git a/src/test/run-pass/macros/macro-at-most-once-rep.rs b/src/test/run-pass/macros/macro-at-most-once-rep-2018.rs similarity index 100% rename from src/test/run-pass/macros/macro-at-most-once-rep.rs rename to src/test/run-pass/macros/macro-at-most-once-rep-2018.rs diff --git a/src/test/ui/issues/issue-39388.rs b/src/test/ui/issues/issue-39388.rs index e5b1cd93614bb..a8e31a648c9ed 100644 --- a/src/test/ui/issues/issue-39388.rs +++ b/src/test/ui/issues/issue-39388.rs @@ -1,7 +1,7 @@ #![allow(unused_macros)] macro_rules! assign { - (($($a:tt)*) = ($($b:tt))*) => { //~ ERROR expected `*` or `+` + (($($a:tt)*) = ($($b:tt))*) => { //~ ERROR expected one of: `*`, `+`, or `?` $($a)* = $($b)* } } diff --git a/src/test/ui/issues/issue-39388.stderr b/src/test/ui/issues/issue-39388.stderr index e04e16e2a0375..62e7dff547692 100644 --- a/src/test/ui/issues/issue-39388.stderr +++ b/src/test/ui/issues/issue-39388.stderr @@ -1,4 +1,4 @@ -error: expected `*` or `+` +error: expected one of: `*`, `+`, or `?` --> $DIR/issue-39388.rs:4:22 | LL | (($($a:tt)*) = ($($b:tt))*) => { diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.rs b/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.rs deleted file mode 100644 index 2d8d2ecf9d7b4..0000000000000 --- a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Test behavior of `?` macro _kleene op_ under the 2015 edition. Namely, it doesn't exist. - -// edition:2015 - -macro_rules! bar { - ($(a)?) => {} //~ERROR expected `*` or `+` -} - -macro_rules! baz { - ($(a),?) => {} //~ERROR expected `*` or `+` -} - -fn main() {} diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.stderr b/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.stderr deleted file mode 100644 index e78f2833078cc..0000000000000 --- a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-rep.stderr +++ /dev/null @@ -1,18 +0,0 @@ -error: expected `*` or `+` - --> $DIR/macro-at-most-once-rep-2015-ques-rep.rs:6:10 - | -LL | ($(a)?) => {} - | ^ - | - = note: `?` is not a macro repetition operator in the 2015 edition, but is accepted in the 2018 edition - -error: expected `*` or `+` - --> $DIR/macro-at-most-once-rep-2015-ques-rep.rs:10:11 - | -LL | ($(a),?) => {} - | ^ - | - = note: `?` is not a macro repetition operator in the 2015 edition, but is accepted in the 2018 edition - -error: aborting due to 2 previous errors - diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.rs b/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.rs deleted file mode 100644 index c8c920ff3f87a..0000000000000 --- a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Test behavior of `?` macro _separator_ under the 2015 edition. Namely, `?` can be used as a -// separator, but you get a migration warning for the edition. - -// edition:2015 -// compile-pass - -#![warn(rust_2018_compatibility)] - -macro_rules! bar { - ($(a)?*) => {} //~WARN using `?` as a separator - //~^WARN this was previously accepted -} - -macro_rules! baz { - ($(a)?+) => {} //~WARN using `?` as a separator - //~^WARN this was previously accepted -} - -fn main() { - bar!(); - bar!(a); - bar!(a?a); - bar!(a?a?a?a?a); - - baz!(a); - baz!(a?a); - baz!(a?a?a?a?a); -} diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.stderr b/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.stderr deleted file mode 100644 index bf1861ae54052..0000000000000 --- a/src/test/ui/macros/macro-at-most-once-rep-2015-ques-sep.stderr +++ /dev/null @@ -1,24 +0,0 @@ -warning: using `?` as a separator is deprecated and will be a hard error in an upcoming edition - --> $DIR/macro-at-most-once-rep-2015-ques-sep.rs:10:10 - | -LL | ($(a)?*) => {} - | ^ - | -note: lint level defined here - --> $DIR/macro-at-most-once-rep-2015-ques-sep.rs:7:9 - | -LL | #![warn(rust_2018_compatibility)] - | ^^^^^^^^^^^^^^^^^^^^^^^ - = note: #[warn(question_mark_macro_sep)] implied by #[warn(rust_2018_compatibility)] - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in the 2018 edition! - = note: for more information, see issue #48075 - -warning: using `?` as a separator is deprecated and will be a hard error in an upcoming edition - --> $DIR/macro-at-most-once-rep-2015-ques-sep.rs:15:10 - | -LL | ($(a)?+) => {} - | ^ - | - = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in the 2018 edition! - = note: for more information, see issue #48075 - diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015.rs b/src/test/ui/macros/macro-at-most-once-rep-2015.rs new file mode 100644 index 0000000000000..e6def0f58c5cd --- /dev/null +++ b/src/test/ui/macros/macro-at-most-once-rep-2015.rs @@ -0,0 +1,41 @@ +// Tests that `?` is a Kleene op and not a macro separator in the 2015 edition. + +// edition:2015 + +macro_rules! foo { + ($(a)?) => {}; +} + +macro_rules! baz { + ($(a),?) => {}; //~ERROR the `?` macro repetition operator +} + +macro_rules! barplus { + ($(a)?+) => {}; // ok. matches "a+" and "+" +} + +macro_rules! barstar { + ($(a)?*) => {}; // ok. matches "a*" and "*" +} + +pub fn main() { + foo!(); + foo!(a); + foo!(a?); //~ ERROR no rules expected the token `?` + foo!(a?a); //~ ERROR no rules expected the token `?` + foo!(a?a?a); //~ ERROR no rules expected the token `?` + + barplus!(); //~ERROR unexpected end of macro invocation + barplus!(a); //~ERROR unexpected end of macro invocation + barplus!(a?); //~ ERROR no rules expected the token `?` + barplus!(a?a); //~ ERROR no rules expected the token `?` + barplus!(a+); + barplus!(+); + + barstar!(); //~ERROR unexpected end of macro invocation + barstar!(a); //~ERROR unexpected end of macro invocation + barstar!(a?); //~ ERROR no rules expected the token `?` + barstar!(a?a); //~ ERROR no rules expected the token `?` + barstar!(a*); + barstar!(*); +} diff --git a/src/test/ui/macros/macro-at-most-once-rep-2015.stderr b/src/test/ui/macros/macro-at-most-once-rep-2015.stderr new file mode 100644 index 0000000000000..81de4f145c8c0 --- /dev/null +++ b/src/test/ui/macros/macro-at-most-once-rep-2015.stderr @@ -0,0 +1,107 @@ +error: the `?` macro repetition operator does not take a separator + --> $DIR/macro-at-most-once-rep-2015.rs:10:10 + | +LL | ($(a),?) => {}; + | ^ + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:24:11 + | +LL | macro_rules! foo { + | ---------------- when calling this macro +... +LL | foo!(a?); + | ^ no rules expected this token in macro call + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:25:11 + | +LL | macro_rules! foo { + | ---------------- when calling this macro +... +LL | foo!(a?a); + | ^ no rules expected this token in macro call + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:26:11 + | +LL | macro_rules! foo { + | ---------------- when calling this macro +... +LL | foo!(a?a?a); + | ^ no rules expected this token in macro call + +error: unexpected end of macro invocation + --> $DIR/macro-at-most-once-rep-2015.rs:28:5 + | +LL | macro_rules! barplus { + | -------------------- when calling this macro +... +LL | barplus!(); + | ^^^^^^^^^^^ missing tokens in macro arguments + +error: unexpected end of macro invocation + --> $DIR/macro-at-most-once-rep-2015.rs:29:15 + | +LL | macro_rules! barplus { + | -------------------- when calling this macro +... +LL | barplus!(a); + | ^ missing tokens in macro arguments + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:30:15 + | +LL | macro_rules! barplus { + | -------------------- when calling this macro +... +LL | barplus!(a?); + | ^ no rules expected this token in macro call + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:31:15 + | +LL | macro_rules! barplus { + | -------------------- when calling this macro +... +LL | barplus!(a?a); + | ^ no rules expected this token in macro call + +error: unexpected end of macro invocation + --> $DIR/macro-at-most-once-rep-2015.rs:35:5 + | +LL | macro_rules! barstar { + | -------------------- when calling this macro +... +LL | barstar!(); + | ^^^^^^^^^^^ missing tokens in macro arguments + +error: unexpected end of macro invocation + --> $DIR/macro-at-most-once-rep-2015.rs:36:15 + | +LL | macro_rules! barstar { + | -------------------- when calling this macro +... +LL | barstar!(a); + | ^ missing tokens in macro arguments + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:37:15 + | +LL | macro_rules! barstar { + | -------------------- when calling this macro +... +LL | barstar!(a?); + | ^ no rules expected this token in macro call + +error: no rules expected the token `?` + --> $DIR/macro-at-most-once-rep-2015.rs:38:15 + | +LL | macro_rules! barstar { + | -------------------- when calling this macro +... +LL | barstar!(a?a); + | ^ no rules expected this token in macro call + +error: aborting due to 12 previous errors + diff --git a/src/test/ui/parser/macro/issue-33569.rs b/src/test/ui/parser/macro/issue-33569.rs index 83f8a562a03ad..9ed53519ceb31 100644 --- a/src/test/ui/parser/macro/issue-33569.rs +++ b/src/test/ui/parser/macro/issue-33569.rs @@ -1,7 +1,7 @@ macro_rules! foo { { $+ } => { //~ ERROR expected identifier, found `+` //~^ ERROR missing fragment specifier - $(x)(y) //~ ERROR expected `*` or `+` + $(x)(y) //~ ERROR expected one of: `*`, `+`, or `?` } } diff --git a/src/test/ui/parser/macro/issue-33569.stderr b/src/test/ui/parser/macro/issue-33569.stderr index 8ba72fc88862a..b4d38d3ce4806 100644 --- a/src/test/ui/parser/macro/issue-33569.stderr +++ b/src/test/ui/parser/macro/issue-33569.stderr @@ -4,7 +4,7 @@ error: expected identifier, found `+` LL | { $+ } => { | ^ -error: expected `*` or `+` +error: expected one of: `*`, `+`, or `?` --> $DIR/issue-33569.rs:4:13 | LL | $(x)(y)