proc_macro: Preserve spans of attributes on functions

This commit updates the tokenization of items which are subsequently passed to
`proc_macro` to ensure that span information is preserved on attributes as much
as possible. Previously this area of the code suffered from rust-lang#43081 where we
haven't actually implemented converting an attribute to to a token tree yet, but
a local fix was possible here.

Closes rust-lang#47941

src/libsyntax/parse/token.rs

@@ -777,11 +777,50 @@ fn prepend_attrs(sess: &ParseSess,
     for attr in attrs {
         assert_eq!(attr.style, ast::AttrStyle::Outer,
                    "inner attributes should prevent cached tokens from existing");
-        // FIXME: Avoid this pretty-print + reparse hack as bove
-        let name = FileName::MacroExpansion;
-        let source = pprust::attr_to_string(attr);
-        let stream = parse_stream_from_source_str(name, source, sess, Some(span));
-        builder.push(stream);
+
+        if attr.is_sugared_doc {
+            let stream = parse_stream_from_source_str(
+                FileName::MacroExpansion,
+                pprust::attr_to_string(attr),
+                sess,
+                Some(span),
+            );
+            builder.push(stream);
+            continue
+        }
+
+        // synthesize # [ $path $tokens ] manually here
+        let mut brackets = tokenstream::TokenStreamBuilder::new();
+
+        // For simple paths, push the identifier directly
+        if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() {
+            let ident = attr.path.segments[0].ident;
+            let token = Ident(ident, ident.as_str().starts_with("r#"));
+            brackets.push(tokenstream::TokenTree::Token(ident.span, token));
+
+        // ... and for more complicated paths, fall back to a reparse hack that
+        // should eventually be removed.
+        } else {
+            let stream = parse_stream_from_source_str(
+                FileName::MacroExpansion,
+                pprust::path_to_string(&attr.path),
+                sess,
+                Some(span),
+            );
+            brackets.push(stream);
+        }
+
+        brackets.push(attr.tokens.clone());
+
+        let tokens = tokenstream::Delimited {
+            delim: DelimToken::Bracket,
+            tts: brackets.build().into(),
+        };
+        // The span we list here for `#` and for `[ ... ]` are both wrong in
+        // that it encompasses more than each token, but it hopefully is "good
+        // enough" for now at least.
+        builder.push(tokenstream::TokenTree::Token(attr.span, Pound));
+        builder.push(tokenstream::TokenTree::Delimited(attr.span, tokens));
     }
     builder.push(tokens.clone());
     Some(builder.build())

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs

@@ -0,0 +1,22 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:attribute-spans-preserved.rs
+
+#![feature(use_extern_macros)]
+
+extern crate attribute_spans_preserved as foo;
+
+use foo::foo;
+
+#[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types
+#[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types
+fn main() {
+}

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr

@@ -0,0 +1,21 @@
+error[E0308]: mismatched types
+  --> $DIR/attribute-spans-preserved.rs:19:23
+   |
+LL | #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types
+   |                       ^^^ expected u32, found reference
+   |
+   = note: expected type `u32`
+              found type `&'static str`
+
+error[E0308]: mismatched types
+  --> $DIR/attribute-spans-preserved.rs:20:21
+   |
+LL | #[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types
+   |                     ^^^ expected u32, found reference
+   |
+   = note: expected type `u32`
+              found type `&'static str`
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0308`.

src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout

@@ -0,0 +1 @@
+fn main (  ) { let y : u32 = "z" ; let x : u32 = "y" ; }

src/test/ui-fulldeps/proc-macro/auxiliary/attribute-spans-preserved.rs

@@ -0,0 +1,44 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+#[proc_macro_attribute]
+pub fn foo(attr: TokenStream, f: TokenStream) -> TokenStream {
+    let mut tokens = f.into_iter();
+    assert_eq!(tokens.next().unwrap().to_string(), "#");
+    let next_attr = match tokens.next().unwrap() {
+        TokenTree::Group(g) => g,
+        _ => panic!(),
+    };
+
+    let fn_tok = tokens.next().unwrap();
+    let ident_tok = tokens.next().unwrap();
+    let args_tok = tokens.next().unwrap();
+    let body = tokens.next().unwrap();
+
+    let new_body = attr.into_iter()
+        .chain(next_attr.stream().into_iter().skip(1));
+
+    let tokens = vec![
+        fn_tok,
+        ident_tok,
+        args_tok,
+        Group::new(Delimiter::Brace, new_body.collect()).into(),
+    ].into_iter().collect::<TokenStream>();
+    println!("{}", tokens);
+    return tokens
+}