Auto merge of rust-lang#50473 - petrochenkov:pmapi, r=alexcrichton

Review proc macro API 1.2

cc rust-lang#38356

Summary of applied changes:
- Documentation for proc macro API 1.2 is expanded.
- Renamed APIs: `Term` -> `Ident`, `TokenTree::Term` -> `TokenTree::Ident`, `Op` -> `Punct`, `TokenTree::Op` -> `TokenTree::Punct`, `Op::op` -> `Punct::as_char`.
- Removed APIs: `Ident::as_str`, use `Display` impl for `Ident` instead.
- New APIs (not stabilized in 1.2): `Ident::new_raw` for creating a raw identifier (I'm not sure `new_x` it's a very idiomatic name though).
- Runtime changes:
    - `Punct::new` now ensures that the input `char` is a valid punctuation character in Rust.
    - `Ident::new` ensures that the input `str` is a valid identifier in Rust.
    - Lifetimes in proc macros are now represented as two joint tokens - `Punct('\'', Spacing::Joint)` and `Ident("lifetime_name_without_quote")` similarly to multi-character operators.
- Stabilized APIs: None yet.

A bit of motivation for renaming (although it was already stated in the review comments):
- With my compiler frontend glasses on `Ident` is the single most appropriate name for this thing, *especially* if we are doing input validation on construction. `TokenTree::Ident` effectively wraps `token::Ident` or `ast::Ident + is_raw`, its meaning is "identifier" and it's already named `ident` in declarative macros.
- Regarding `Punct`, the motivation is that `Op` is actively misleading. The thing doesn't mean an operator, it's neither a subset of operators (there is non-operator punctuation in the language), nor superset (operators can be multicharacter while this thing is always a single character). So I named it `Punct` (first proposed in [the original RFC](rust-lang/rfcs#1566), then [by @SimonSapin](rust-lang#38356 (comment))) , together with input validation it's now a subset of ASCII punctuation character category (`u8::is_ascii_punctuation`).

Author: bors

src/libproc_macro/lib.rs

@@ -14,10 +14,11 @@
 //! This quasiquoter uses macros 2.0 hygiene to reliably access
 //! items from `proc_macro`, to build a `proc_macro::TokenStream`.
 
-use {Delimiter, Literal, Spacing, Span, Term, Op, Group, TokenStream, TokenTree};
+use {Delimiter, Literal, Spacing, Span, Ident, Punct, Group, TokenStream, TokenTree};
 
 use syntax::ext::base::{ExtCtxt, ProcMacro};
 use syntax::parse::token;
+use syntax::symbol::Symbol;
 use syntax::tokenstream;
 
 pub struct Quoter;
@@ -35,14 +36,14 @@ macro_rules! tt2ts {
 }
 
 macro_rules! quote_tok {
-    (,) => { tt2ts!(Op::new(',', Spacing::Alone)) };
-    (.) => { tt2ts!(Op::new('.', Spacing::Alone)) };
-    (:) => { tt2ts!(Op::new(':', Spacing::Alone)) };
-    (|) => { tt2ts!(Op::new('|', Spacing::Alone)) };
+    (,) => { tt2ts!(Punct::new(',', Spacing::Alone)) };
+    (.) => { tt2ts!(Punct::new('.', Spacing::Alone)) };
+    (:) => { tt2ts!(Punct::new(':', Spacing::Alone)) };
+    (|) => { tt2ts!(Punct::new('|', Spacing::Alone)) };
     (::) => {
         [
-            TokenTree::from(Op::new(':', Spacing::Joint)),
-            TokenTree::from(Op::new(':', Spacing::Alone)),
+            TokenTree::from(Punct::new(':', Spacing::Joint)),
+            TokenTree::from(Punct::new(':', Spacing::Alone)),
         ].iter()
             .cloned()
             .map(|mut x| {
@@ -51,13 +52,13 @@ macro_rules! quote_tok {
             })
             .collect::<TokenStream>()
     };
-    (!) => { tt2ts!(Op::new('!', Spacing::Alone)) };
-    (<) => { tt2ts!(Op::new('<', Spacing::Alone)) };
-    (>) => { tt2ts!(Op::new('>', Spacing::Alone)) };
-    (_) => { tt2ts!(Op::new('_', Spacing::Alone)) };
+    (!) => { tt2ts!(Punct::new('!', Spacing::Alone)) };
+    (<) => { tt2ts!(Punct::new('<', Spacing::Alone)) };
+    (>) => { tt2ts!(Punct::new('>', Spacing::Alone)) };
+    (_) => { tt2ts!(Punct::new('_', Spacing::Alone)) };
     (0) => { tt2ts!(Literal::i8_unsuffixed(0)) };
-    (&) => { tt2ts!(Op::new('&', Spacing::Alone)) };
-    ($i:ident) => { tt2ts!(Term::new(stringify!($i), Span::def_site())) };
+    (&) => { tt2ts!(Punct::new('&', Spacing::Alone)) };
+    ($i:ident) => { tt2ts!(Ident::new(stringify!($i), Span::def_site())) };
 }
 
 macro_rules! quote_tree {
@@ -110,15 +111,15 @@ impl Quote for TokenStream {
             if after_dollar {
                 after_dollar = false;
                 match tree {
-                    TokenTree::Term(_) => {
+                    TokenTree::Ident(_) => {
                         let tree = TokenStream::from(tree);
                         return Some(quote!(::__internal::unquote(&(unquote tree)),));
                     }
-                    TokenTree::Op(ref tt) if tt.op() == '$' => {}
+                    TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
                     _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
                 }
-            } else if let TokenTree::Op(tt) = tree {
-                if tt.op() == '$' {
+            } else if let TokenTree::Punct(ref tt) = tree {
+                if tt.as_char() == '$' {
                     after_dollar = true;
                     return None;
                 }
@@ -143,9 +144,9 @@ impl Quote for TokenStream {
 impl Quote for TokenTree {
     fn quote(self) -> TokenStream {
         match self {
-            TokenTree::Op(tt) => quote!(::TokenTree::Op( (quote tt) )),
+            TokenTree::Punct(tt) => quote!(::TokenTree::Punct( (quote tt) )),
             TokenTree::Group(tt) => quote!(::TokenTree::Group( (quote tt) )),
-            TokenTree::Term(tt) => quote!(::TokenTree::Term( (quote tt) )),
+            TokenTree::Ident(tt) => quote!(::TokenTree::Ident( (quote tt) )),
             TokenTree::Literal(tt) => quote!(::TokenTree::Literal( (quote tt) )),
         }
     }
@@ -175,15 +176,15 @@ impl Quote for Group {
     }
 }
 
-impl Quote for Op {
+impl Quote for Punct {
     fn quote(self) -> TokenStream {
-        quote!(::Op::new((quote self.op()), (quote self.spacing())))
+        quote!(::Punct::new((quote self.as_char()), (quote self.spacing())))
     }
 }
 
-impl Quote for Term {
+impl Quote for Ident {
     fn quote(self) -> TokenStream {
-        quote!(::Term::new((quote self.sym.as_str()), (quote self.span())))
+        quote!(::Ident::new((quote self.sym.as_str()), (quote self.span())))
     }
 }
 
@@ -195,14 +196,32 @@ impl Quote for Span {
 
 macro_rules! literals {
     ($($i:ident),*; $($raw:ident),*) => {
+        pub struct SpannedSymbol {
+            sym: Symbol,
+            span: Span,
+        }
+
+        impl SpannedSymbol {
+            pub fn new(string: &str, span: Span) -> SpannedSymbol {
+                SpannedSymbol { sym: Symbol::intern(string), span }
+            }
+        }
+
+        impl Quote for SpannedSymbol {
+            fn quote(self) -> TokenStream {
+                quote!(::__internal::SpannedSymbol::new((quote self.sym.as_str()),
+                                                        (quote self.span)))
+            }
+        }
+
         pub enum LiteralKind {
             $($i,)*
             $($raw(u16),)*
         }
 
         impl LiteralKind {
-            pub fn with_contents_and_suffix(self, contents: Term, suffix: Option<Term>)
-                                            -> Literal {
+            pub fn with_contents_and_suffix(self, contents: SpannedSymbol,
+                                            suffix: Option<SpannedSymbol>) -> Literal {
                 let sym = contents.sym;
                 let suffix = suffix.map(|t| t.sym);
                 match self {
@@ -225,13 +244,14 @@ macro_rules! literals {
         }
 
         impl Literal {
-            fn kind_contents_and_suffix(self) -> (LiteralKind, Term, Option<Term>) {
+            fn kind_contents_and_suffix(self) -> (LiteralKind, SpannedSymbol, Option<SpannedSymbol>)
+            {
                 let (kind, contents) = match self.lit {
                     $(token::Lit::$i(contents) => (LiteralKind::$i, contents),)*
                     $(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)*
                 };
-                let suffix = self.suffix.map(|sym| Term::new(&sym.as_str(), self.span()));
-                (kind, Term::new(&contents.as_str(), self.span()), suffix)
+                let suffix = self.suffix.map(|sym| SpannedSymbol::new(&sym.as_str(), self.span()));
+                (kind, SpannedSymbol::new(&contents.as_str(), self.span()), suffix)
             }
         }
 

src/libproc_macro/quote.rs

@@ -314,6 +314,7 @@ fn hash_token<'a, 'gcx, W: StableHasherResult>(
         token::Token::Pound |
         token::Token::Dollar |
         token::Token::Question |
+        token::Token::SingleQuote |
         token::Token::Whitespace |
         token::Token::Comment |
         token::Token::Eof => {}

src/librustc/ich/impls_syntax.rs

@@ -353,7 +353,7 @@ impl<'a> Classifier<'a> {
             token::Lifetime(..) => Class::Lifetime,
 
             token::Eof | token::Interpolated(..) |
-            token::Tilde | token::At | token::DotEq => Class::None,
+            token::Tilde | token::At | token::DotEq | token::SingleQuote => Class::None,
         };
 
         // Anything that didn't return above is the simple case where we the

src/librustdoc/html/highlight.rs

@@ -711,6 +711,7 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
         token::Pound        => "Pound",
         token::Dollar       => "Dollar",
         token::Question     => "Question",
+        token::SingleQuote  => "SingleQuote",
         token::Eof          => "Eof",
 
         token::Whitespace | token::Comment | token::Shebang(_) => {

src/libsyntax/ext/quote.rs

@@ -1770,6 +1770,12 @@ fn ident_continue(c: Option<char>) -> bool {
     (c > '\x7f' && c.is_xid_continue())
 }
 
+// The string is a valid identifier or a lifetime identifier.
+pub fn is_valid_ident(s: &str) -> bool {
+    let mut chars = s.chars();
+    ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch)))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/libsyntax/parse/lexer/mod.rs

@@ -210,6 +210,8 @@ pub enum Token {
     Pound,
     Dollar,
     Question,
+    /// Used by proc macros for representing lifetimes, not generated by lexer right now.
+    SingleQuote,
     /// An opening delimiter, eg. `{`
     OpenDelim(DelimToken),
     /// A closing delimiter, eg. `}`
@@ -513,6 +515,10 @@ impl Token {
                 Colon => ModSep,
                 _ => return None,
             },
+            SingleQuote => match joint {
+                Ident(ident, false) => Lifetime(ident),
+                _ => return None,
+            },
 
             Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | DotEq |
             DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar |

src/libsyntax/parse/token.rs

@@ -224,6 +224,7 @@ pub fn token_to_string(tok: &Token) -> String {
         token::Pound                => "#".to_string(),
         token::Dollar               => "$".to_string(),
         token::Question             => "?".to_string(),
+        token::SingleQuote          => "'".to_string(),
 
         /* Literals */
         token::Literal(lit, suf) => {

src/libsyntax/print/pprust.rs

@@ -53,7 +53,7 @@ pub fn bar(attr: TokenStream, input: TokenStream) -> TokenStream {
 
 fn assert_inline(slice: &mut &[TokenTree]) {
     match &slice[0] {
-        TokenTree::Op(tt) => assert_eq!(tt.op(), '#'),
+        TokenTree::Punct(tt) => assert_eq!(tt.as_char(), '#'),
         _ => panic!("expected '#' char"),
     }
     match &slice[1] {
@@ -65,8 +65,8 @@ fn assert_inline(slice: &mut &[TokenTree]) {
 
 fn assert_doc(slice: &mut &[TokenTree]) {
     match &slice[0] {
-        TokenTree::Op(tt) => {
-            assert_eq!(tt.op(), '#');
+        TokenTree::Punct(tt) => {
+            assert_eq!(tt.as_char(), '#');
             assert_eq!(tt.spacing(), Spacing::Alone);
         }
         _ => panic!("expected #"),
@@ -86,12 +86,12 @@ fn assert_doc(slice: &mut &[TokenTree]) {
     }
 
     match &tokens[0] {
-        TokenTree::Term(tt) => assert_eq!("doc", &*tt.to_string()),
+        TokenTree::Ident(tt) => assert_eq!("doc", &*tt.to_string()),
         _ => panic!("expected `doc`"),
     }
     match &tokens[1] {
-        TokenTree::Op(tt) => {
-            assert_eq!(tt.op(), '=');
+        TokenTree::Punct(tt) => {
+            assert_eq!(tt.as_char(), '=');
             assert_eq!(tt.spacing(), Spacing::Alone);
         }
         _ => panic!("expected equals"),
@@ -106,7 +106,7 @@ fn assert_doc(slice: &mut &[TokenTree]) {
 
 fn assert_invoc(slice: &mut &[TokenTree]) {
     match &slice[0] {
-        TokenTree::Op(tt) => assert_eq!(tt.op(), '#'),
+        TokenTree::Punct(tt) => assert_eq!(tt.as_char(), '#'),
         _ => panic!("expected '#' char"),
     }
     match &slice[1] {
@@ -118,11 +118,11 @@ fn assert_invoc(slice: &mut &[TokenTree]) {
 
 fn assert_foo(slice: &mut &[TokenTree]) {
     match &slice[0] {
-        TokenTree::Term(tt) => assert_eq!(&*tt.to_string(), "fn"),
+        TokenTree::Ident(tt) => assert_eq!(&*tt.to_string(), "fn"),
         _ => panic!("expected fn"),
     }
     match &slice[1] {
-        TokenTree::Term(tt) => assert_eq!(&*tt.to_string(), "foo"),
+        TokenTree::Ident(tt) => assert_eq!(&*tt.to_string(), "foo"),
         _ => panic!("expected foo"),
     }
     match &slice[2] {
@@ -148,8 +148,8 @@ fn fold_tree(input: TokenTree) -> TokenTree {
         TokenTree::Group(b) => {
             TokenTree::Group(Group::new(b.delimiter(), fold_stream(b.stream())))
         }
-        TokenTree::Op(b) => TokenTree::Op(b),
-        TokenTree::Term(a) => TokenTree::Term(a),
+        TokenTree::Punct(b) => TokenTree::Punct(b),
+        TokenTree::Ident(a) => TokenTree::Ident(a),
         TokenTree::Literal(a) => {
             if a.to_string() != "\"foo\"" {
                 TokenTree::Literal(a)

src/test/compile-fail-fulldeps/proc-macro/auxiliary/attributes-included.rs

@@ -11,7 +11,6 @@
 // force-host
 // no-prefer-dynamic
 
-#![feature(proc_macro, proc_macro_lib)]
 #![crate_type = "proc-macro"]
 
 extern crate proc_macro;

src/test/compile-fail-fulldeps/proc-macro/auxiliary/issue_38586.rs

@@ -11,8 +11,6 @@
 // aux-build:issue_38586.rs
 // ignore-stage1
 
-#![feature(proc_macro)]
-
 #[macro_use]
 extern crate issue_38586;
 

src/test/compile-fail-fulldeps/proc-macro/issue-38586.rs

@@ -11,7 +11,7 @@
 // aux-build:bang_proc_macro2.rs
 // ignore-stage1
 
-#![feature(proc_macro, proc_macro_non_items)]
+#![feature(use_extern_macros, proc_macro_non_items)]
 #![allow(unused_macros)]
 
 extern crate bang_proc_macro2;

src/test/compile-fail-fulldeps/proc-macro/lints_in_proc_macros.rs

@@ -10,7 +10,7 @@
 
 // aux-build:bang_proc_macro.rs
 
-#![feature(proc_macro, proc_macro_non_items)]
+#![feature(proc_macro_non_items)]
 
 #[macro_use]
 extern crate bang_proc_macro;

src/test/compile-fail-fulldeps/proc-macro/macro-use-bang.rs

@@ -10,7 +10,7 @@
 
 // aux-build:proc-macro-gates.rs
 
-#![feature(proc_macro, stmt_expr_attributes)]
+#![feature(use_extern_macros, stmt_expr_attributes)]
 
 extern crate proc_macro_gates as foo;
 

src/test/compile-fail-fulldeps/proc-macro/proc-macro-gates2.rs

@@ -33,7 +33,7 @@ pub fn cond(input: TokenStream) -> TokenStream {
             panic!("Invalid macro usage in cond: {}", cond);
         }
         let is_else = match test {
-            TokenTree::Term(word) => &*word.to_string() == "else",
+            TokenTree::Ident(ref word) => &*word.to_string() == "else",
             _ => false,
         };
         conds.push(if is_else || input.peek().is_none() {

src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs

@@ -11,7 +11,7 @@
 // no-prefer-dynamic
 
 #![crate_type = "proc-macro"]
-#![feature(proc_macro, proc_macro_lib, proc_macro_non_items)]
+#![feature(proc_macro, proc_macro_non_items)]
 
 extern crate proc_macro;
 

src/test/run-pass-fulldeps/auxiliary/hello_macro.rs

@@ -11,7 +11,7 @@
 // no-prefer-dynamic
 
 #![crate_type = "proc-macro"]
-#![feature(proc_macro, proc_macro_lib, proc_macro_non_items)]
+#![feature(proc_macro, proc_macro_non_items)]
 
 extern crate proc_macro;
 

src/test/run-pass-fulldeps/auxiliary/proc_macro_def.rs

@@ -11,7 +11,7 @@
 // aux-build:cond_plugin.rs
 // ignore-stage1
 
-#![feature(proc_macro, proc_macro_non_items)]
+#![feature(use_extern_macros, proc_macro_non_items)]
 
 extern crate cond_plugin;
 

src/test/run-pass-fulldeps/macro-quote-cond.rs

@@ -13,7 +13,7 @@
 // aux-build:hello_macro.rs
 // ignore-stage1
 
-#![feature(proc_macro, proc_macro_non_items)]
+#![feature(use_extern_macros, proc_macro_non_items)]
 
 extern crate hello_macro;
 

src/test/run-pass-fulldeps/macro-quote-test.rs

@@ -28,7 +28,7 @@ fn count_compound_ops_helper(input: TokenStream) -> u32 {
     let mut count = 0;
     for token in input {
         match &token {
-            TokenTree::Op(tt) if tt.spacing() == Spacing::Alone => {
+            TokenTree::Punct(tt) if tt.spacing() == Spacing::Alone => {
                 count += 1;
             }
             TokenTree::Group(tt) => {