From 53323751a9caaf678689e0d437f79d0c169b2dae Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 19 Jul 2018 07:06:31 -0700 Subject: [PATCH] proc_macro: Preserve spans of attributes on functions This commit updates the tokenization of items which are subsequently passed to `proc_macro` to ensure that span information is preserved on attributes as much as possible. Previously this area of the code suffered from #43081 where we haven't actually implemented converting an attribute to to a token tree yet, but a local fix was possible here. Closes #47941 --- src/libsyntax/parse/token.rs | 49 +++++++++++++++++-- .../proc-macro/attribute-spans-preserved.rs | 22 +++++++++ .../attribute-spans-preserved.stderr | 21 ++++++++ .../attribute-spans-preserved.stdout | 1 + .../auxiliary/attribute-spans-preserved.rs | 44 +++++++++++++++++ 5 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs create mode 100644 src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr create mode 100644 src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout create mode 100644 src/test/ui-fulldeps/proc-macro/auxiliary/attribute-spans-preserved.rs diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index aef3beeccdf9c..fd8f394a600f0 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -777,11 +777,50 @@ fn prepend_attrs(sess: &ParseSess, for attr in attrs { assert_eq!(attr.style, ast::AttrStyle::Outer, "inner attributes should prevent cached tokens from existing"); - // FIXME: Avoid this pretty-print + reparse hack as bove - let name = FileName::MacroExpansion; - let source = pprust::attr_to_string(attr); - let stream = parse_stream_from_source_str(name, source, sess, Some(span)); - builder.push(stream); + + if attr.is_sugared_doc { + let stream = parse_stream_from_source_str( + FileName::MacroExpansion, + pprust::attr_to_string(attr), + sess, + Some(span), + ); + builder.push(stream); + continue + } + + // synthesize # [ $path $tokens ] manually here + let mut brackets = tokenstream::TokenStreamBuilder::new(); + + // For simple paths, push the identifier directly + if attr.path.segments.len() == 1 && attr.path.segments[0].args.is_none() { + let ident = attr.path.segments[0].ident; + let token = Ident(ident, ident.as_str().starts_with("r#")); + brackets.push(tokenstream::TokenTree::Token(ident.span, token)); + + // ... and for more complicated paths, fall back to a reparse hack that + // should eventually be removed. + } else { + let stream = parse_stream_from_source_str( + FileName::MacroExpansion, + pprust::path_to_string(&attr.path), + sess, + Some(span), + ); + brackets.push(stream); + } + + brackets.push(attr.tokens.clone()); + + let tokens = tokenstream::Delimited { + delim: DelimToken::Bracket, + tts: brackets.build().into(), + }; + // The span we list here for `#` and for `[ ... ]` are both wrong in + // that it encompasses more than each token, but it hopefully is "good + // enough" for now at least. + builder.push(tokenstream::TokenTree::Token(attr.span, Pound)); + builder.push(tokenstream::TokenTree::Delimited(attr.span, tokens)); } builder.push(tokens.clone()); Some(builder.build()) diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs new file mode 100644 index 0000000000000..e1401653ba356 --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs @@ -0,0 +1,22 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// aux-build:attribute-spans-preserved.rs + +#![feature(use_extern_macros)] + +extern crate attribute_spans_preserved as foo; + +use foo::foo; + +#[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types +#[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types +fn main() { +} diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr new file mode 100644 index 0000000000000..fe62bd23b87c9 --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr @@ -0,0 +1,21 @@ +error[E0308]: mismatched types + --> $DIR/attribute-spans-preserved.rs:19:23 + | +LL | #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types + | ^^^ expected u32, found reference + | + = note: expected type `u32` + found type `&'static str` + +error[E0308]: mismatched types + --> $DIR/attribute-spans-preserved.rs:20:21 + | +LL | #[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types + | ^^^ expected u32, found reference + | + = note: expected type `u32` + found type `&'static str` + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0308`. diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout new file mode 100644 index 0000000000000..33dc064ef680f --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout @@ -0,0 +1 @@ +fn main ( ) { let y : u32 = "z" ; let x : u32 = "y" ; } diff --git a/src/test/ui-fulldeps/proc-macro/auxiliary/attribute-spans-preserved.rs b/src/test/ui-fulldeps/proc-macro/auxiliary/attribute-spans-preserved.rs new file mode 100644 index 0000000000000..e725cc7afb82b --- /dev/null +++ b/src/test/ui-fulldeps/proc-macro/auxiliary/attribute-spans-preserved.rs @@ -0,0 +1,44 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::*; + +#[proc_macro_attribute] +pub fn foo(attr: TokenStream, f: TokenStream) -> TokenStream { + let mut tokens = f.into_iter(); + assert_eq!(tokens.next().unwrap().to_string(), "#"); + let next_attr = match tokens.next().unwrap() { + TokenTree::Group(g) => g, + _ => panic!(), + }; + + let fn_tok = tokens.next().unwrap(); + let ident_tok = tokens.next().unwrap(); + let args_tok = tokens.next().unwrap(); + let body = tokens.next().unwrap(); + + let new_body = attr.into_iter() + .chain(next_attr.stream().into_iter().skip(1)); + + let tokens = vec![ + fn_tok, + ident_tok, + args_tok, + Group::new(Delimiter::Brace, new_body.collect()).into(), + ].into_iter().collect::(); + println!("{}", tokens); + return tokens +}