From 46f48d31fed06295a83f7695b74159ff58a0943c Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Tue, 21 Jul 2020 22:16:19 +0300 Subject: [PATCH 1/5] rustc_ast: Stop using "string typing" for doc comment tokens Explicitly store their kind and style retrieved during lexing in the token --- src/librustc_ast/ast.rs | 4 +- src/librustc_ast/attr/mod.rs | 25 +++--- src/librustc_ast/mut_visit.rs | 2 +- src/librustc_ast/token.rs | 13 ++- src/librustc_ast/util/comments.rs | 107 +++++++++++------------ src/librustc_ast/util/comments/tests.rs | 42 ++++----- src/librustc_ast/visit.rs | 2 +- src/librustc_ast_lowering/lib.rs | 2 +- src/librustc_ast_pretty/pprust.rs | 27 ++++-- src/librustc_expand/parse/lexer/tests.rs | 16 ++-- src/librustc_expand/parse/tests.rs | 6 +- src/librustc_expand/proc_macro_server.rs | 7 +- src/librustc_parse/lexer/mod.rs | 21 ++--- src/librustc_parse/lib.rs | 6 +- src/librustc_parse/parser/attr.rs | 15 ++-- src/librustc_parse/parser/diagnostics.rs | 2 +- src/librustc_parse/parser/item.rs | 4 +- src/librustc_parse/parser/mod.rs | 12 +-- src/librustc_save_analysis/lib.rs | 4 +- src/librustdoc/clean/types.rs | 14 +-- 20 files changed, 172 insertions(+), 159 deletions(-) diff --git a/src/librustc_ast/ast.rs b/src/librustc_ast/ast.rs index 6543117774a68..9337b27e5e9f0 100644 --- a/src/librustc_ast/ast.rs +++ b/src/librustc_ast/ast.rs @@ -23,7 +23,7 @@ pub use GenericArgs::*; pub use UnsafeSource::*; use crate::ptr::P; -use crate::token::{self, DelimToken}; +use crate::token::{self, CommentKind, DelimToken}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -2365,7 +2365,7 @@ pub enum AttrKind { /// A doc comment (e.g. `/// ...`, `//! ...`, `/** ... */`, `/*! ... */`). /// Doc attributes (e.g. `#[doc="..."]`) are represented with the `Normal` /// variant (which is much less compact and thus more expensive). - DocComment(Symbol), + DocComment(CommentKind, Symbol), } /// `TraitRef`s appear in impls. diff --git a/src/librustc_ast/attr/mod.rs b/src/librustc_ast/attr/mod.rs index 809fda865422a..847d126b3efc0 100644 --- a/src/librustc_ast/attr/mod.rs +++ b/src/librustc_ast/attr/mod.rs @@ -7,7 +7,7 @@ use crate::ast::{MacArgs, MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem}; use crate::ast::{Path, PathSegment}; use crate::mut_visit::visit_clobber; use crate::ptr::P; -use crate::token::{self, Token}; +use crate::token::{self, CommentKind, Token}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use rustc_data_structures::sync::Lock; @@ -169,7 +169,7 @@ impl Attribute { pub fn has_name(&self, name: Symbol) -> bool { match self.kind { AttrKind::Normal(ref item) => item.path == name, - AttrKind::DocComment(_) => false, + AttrKind::DocComment(..) => false, } } @@ -198,7 +198,7 @@ impl Attribute { None } } - AttrKind::DocComment(_) => None, + AttrKind::DocComment(..) => None, } } pub fn name_or_empty(&self) -> Symbol { @@ -218,7 +218,7 @@ impl Attribute { Some(MetaItem { kind: MetaItemKind::List(list), .. }) => Some(list), _ => None, }, - AttrKind::DocComment(_) => None, + AttrKind::DocComment(..) => None, } } @@ -314,13 +314,13 @@ impl Attribute { pub fn is_doc_comment(&self) -> bool { match self.kind { AttrKind::Normal(_) => false, - AttrKind::DocComment(_) => true, + AttrKind::DocComment(..) => true, } } pub fn doc_str(&self) -> Option { match self.kind { - AttrKind::DocComment(symbol) => Some(symbol), + AttrKind::DocComment(.., data) => Some(data), AttrKind::Normal(ref item) if item.path == sym::doc => { item.meta(self.span).and_then(|meta| meta.value_str()) } @@ -331,14 +331,14 @@ impl Attribute { pub fn get_normal_item(&self) -> &AttrItem { match self.kind { AttrKind::Normal(ref item) => item, - AttrKind::DocComment(_) => panic!("unexpected doc comment"), + AttrKind::DocComment(..) => panic!("unexpected doc comment"), } } pub fn unwrap_normal_item(self) -> AttrItem { match self.kind { AttrKind::Normal(item) => item, - AttrKind::DocComment(_) => panic!("unexpected doc comment"), + AttrKind::DocComment(..) => panic!("unexpected doc comment"), } } @@ -405,8 +405,13 @@ pub fn mk_attr_outer(item: MetaItem) -> Attribute { mk_attr(AttrStyle::Outer, item.path, item.kind.mac_args(item.span), item.span) } -pub fn mk_doc_comment(style: AttrStyle, comment: Symbol, span: Span) -> Attribute { - Attribute { kind: AttrKind::DocComment(comment), id: mk_attr_id(), style, span } +pub fn mk_doc_comment( + comment_kind: CommentKind, + style: AttrStyle, + data: Symbol, + span: Span, +) -> Attribute { + Attribute { kind: AttrKind::DocComment(comment_kind, data), id: mk_attr_id(), style, span } } pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool { diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs index 54f81ef106fe1..df6e8218f6c3a 100644 --- a/src/librustc_ast/mut_visit.rs +++ b/src/librustc_ast/mut_visit.rs @@ -582,7 +582,7 @@ pub fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { vis.visit_path(path); visit_mac_args(args, vis); } - AttrKind::DocComment(_) => {} + AttrKind::DocComment(..) => {} } vis.visit_span(span); } diff --git a/src/librustc_ast/token.rs b/src/librustc_ast/token.rs index e1c94ddf78288..bcce881ed48c5 100644 --- a/src/librustc_ast/token.rs +++ b/src/librustc_ast/token.rs @@ -17,6 +17,12 @@ use rustc_span::{self, Span, DUMMY_SP}; use std::borrow::Cow; use std::{fmt, mem}; +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] +pub enum CommentKind { + Line, + Block, +} + #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] #[derive(HashStable_Generic)] pub enum BinOpToken { @@ -238,9 +244,10 @@ pub enum TokenKind { Interpolated(Lrc), - // Can be expanded into several tokens. - /// A doc comment. - DocComment(Symbol), + /// A doc comment token. + /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) + /// similarly to symbols in string literal tokens. + DocComment(CommentKind, ast::AttrStyle, Symbol), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for diff --git a/src/librustc_ast/util/comments.rs b/src/librustc_ast/util/comments.rs index 39921b2022606..543c22f4def05 100644 --- a/src/librustc_ast/util/comments.rs +++ b/src/librustc_ast/util/comments.rs @@ -1,11 +1,10 @@ pub use CommentStyle::*; -use crate::ast; +use crate::ast::AttrStyle; +use crate::token::CommentKind; use rustc_span::source_map::SourceMap; use rustc_span::{BytePos, CharPos, FileName, Pos, Symbol}; -use log::debug; - #[cfg(test)] mod tests; @@ -28,43 +27,46 @@ pub struct Comment { pub pos: BytePos, } -pub fn is_line_doc_comment(s: &str) -> bool { - let res = (s.starts_with("///") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'/') - || s.starts_with("//!"); - debug!("is {:?} a doc comment? {}", s, res); - res -} - -pub fn is_block_doc_comment(s: &str) -> bool { - // Prevent `/**/` from being parsed as a doc comment - let res = ((s.starts_with("/**") && *s.as_bytes().get(3).unwrap_or(&b' ') != b'*') - || s.starts_with("/*!")) - && s.len() >= 5; - debug!("is {:?} a doc comment? {}", s, res); - res -} - -// FIXME(#64197): Try to privatize this again. -pub fn is_doc_comment(s: &str) -> bool { - (s.starts_with("///") && is_line_doc_comment(s)) - || s.starts_with("//!") - || (s.starts_with("/**") && is_block_doc_comment(s)) - || s.starts_with("/*!") +/// For a full line comment string returns its doc comment style if it's a doc comment +/// and returns `None` if it's a regular comment. +pub fn line_doc_comment_style(line_comment: &str) -> Option { + let line_comment = line_comment.as_bytes(); + assert!(line_comment.starts_with(b"//")); + match line_comment.get(2) { + // `//!` is an inner line doc comment. + Some(b'!') => Some(AttrStyle::Inner), + Some(b'/') => match line_comment.get(3) { + // `////` (more than 3 slashes) is not considered a doc comment. + Some(b'/') => None, + // Otherwise `///` is an outer line doc comment. + _ => Some(AttrStyle::Outer), + }, + _ => None, + } } -pub fn doc_comment_style(comment: Symbol) -> ast::AttrStyle { - let comment = &comment.as_str(); - assert!(is_doc_comment(comment)); - if comment.starts_with("//!") || comment.starts_with("/*!") { - ast::AttrStyle::Inner - } else { - ast::AttrStyle::Outer +/// For a full block comment string returns its doc comment style if it's a doc comment +/// and returns `None` if it's a regular comment. +pub fn block_doc_comment_style(block_comment: &str, terminated: bool) -> Option { + let block_comment = block_comment.as_bytes(); + assert!(block_comment.starts_with(b"/*")); + assert!(!terminated || block_comment.ends_with(b"*/")); + match block_comment.get(2) { + // `/*!` is an inner block doc comment. + Some(b'!') => Some(AttrStyle::Inner), + Some(b'*') => match block_comment.get(3) { + // `/***` (more than 2 stars) is not considered a doc comment. + Some(b'*') => None, + // `/**/` is not considered a doc comment. + Some(b'/') if block_comment.len() == 4 => None, + // Otherwise `/**` is an outer block doc comment. + _ => Some(AttrStyle::Outer), + }, + _ => None, } } -pub fn strip_doc_comment_decoration(comment: Symbol) -> String { - let comment = &comment.as_str(); - +pub fn strip_doc_comment_decoration(data: Symbol, comment_kind: CommentKind) -> String { /// remove whitespace-only lines from the start/end of lines fn vertical_trim(lines: Vec) -> Vec { let mut i = 0; @@ -126,26 +128,19 @@ pub fn strip_doc_comment_decoration(comment: Symbol) -> String { } } - // one-line comments lose their prefix - const ONELINERS: &[&str] = &["///!", "///", "//!", "//"]; - - for prefix in ONELINERS { - if comment.starts_with(*prefix) { - return (&comment[prefix.len()..]).to_string(); + match comment_kind { + CommentKind::Line => { + let data = data.as_str(); + let prefix_len = if data.starts_with('!') { 1 } else { 0 }; + data[prefix_len..].to_string() + } + CommentKind::Block => { + let lines = data.as_str().lines().map(|s| s.to_string()).collect::>(); + let lines = vertical_trim(lines); + let lines = horizontal_trim(lines); + lines.join("\n") } } - - if comment.starts_with("/*") { - let lines = - comment[3..comment.len() - 2].lines().map(|s| s.to_string()).collect::>(); - - let lines = vertical_trim(lines); - let lines = horizontal_trim(lines); - - return lines.join("\n"); - } - - panic!("not a doc-comment: {}", comment); } /// Returns `None` if the first `col` chars of `s` contain a non-whitespace char. @@ -226,8 +221,8 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec { - if !is_block_doc_comment(token_text) { + rustc_lexer::TokenKind::BlockComment { terminated } => { + if block_doc_comment_style(token_text, terminated).is_none() { let code_to_the_right = match text[pos + token.len..].chars().next() { Some('\r' | '\n') => false, _ => true, @@ -249,7 +244,7 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec { - if !is_doc_comment(token_text) { + if line_doc_comment_style(token_text).is_none() { comments.push(Comment { style: if code_to_the_left { Trailing } else { Isolated }, lines: vec![token_text.to_string()], diff --git a/src/librustc_ast/util/comments/tests.rs b/src/librustc_ast/util/comments/tests.rs index f08011fe4f862..274e9ed39bb28 100644 --- a/src/librustc_ast/util/comments/tests.rs +++ b/src/librustc_ast/util/comments/tests.rs @@ -1,11 +1,18 @@ use super::*; use crate::with_default_session_globals; +#[test] +fn line_doc_comments() { + assert!(line_doc_comment_style("///").is_some()); + assert!(line_doc_comment_style("/// blah").is_some()); + assert!(line_doc_comment_style("////").is_none()); +} + #[test] fn test_block_doc_comment_1() { with_default_session_globals(|| { - let comment = "/**\n * Test \n ** Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment)); + let comment = "\n * Test \n ** Test\n * Test\n"; + let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); assert_eq!(stripped, " Test \n* Test\n Test"); }) } @@ -13,8 +20,8 @@ fn test_block_doc_comment_1() { #[test] fn test_block_doc_comment_2() { with_default_session_globals(|| { - let comment = "/**\n * Test\n * Test\n*/"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment)); + let comment = "\n * Test\n * Test\n"; + let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); assert_eq!(stripped, " Test\n Test"); }) } @@ -22,37 +29,22 @@ fn test_block_doc_comment_2() { #[test] fn test_block_doc_comment_3() { with_default_session_globals(|| { - let comment = "/**\n let a: *i32;\n *a = 5;\n*/"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment)); + let comment = "\n let a: *i32;\n *a = 5;\n"; + let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); assert_eq!(stripped, " let a: *i32;\n *a = 5;"); }) } -#[test] -fn test_block_doc_comment_4() { - with_default_session_globals(|| { - let comment = "/*******************\n test\n *********************/"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment)); - assert_eq!(stripped, " test"); - }) -} - #[test] fn test_line_doc_comment() { with_default_session_globals(|| { - let stripped = strip_doc_comment_decoration(Symbol::intern("/// test")); + let stripped = strip_doc_comment_decoration(Symbol::intern(" test"), CommentKind::Line); assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("///! test")); + let stripped = strip_doc_comment_decoration(Symbol::intern("! test"), CommentKind::Line); assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("// test")); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("// test")); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("///test")); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("///!test")); + let stripped = strip_doc_comment_decoration(Symbol::intern("test"), CommentKind::Line); assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("//test")); + let stripped = strip_doc_comment_decoration(Symbol::intern("!test"), CommentKind::Line); assert_eq!(stripped, "test"); }) } diff --git a/src/librustc_ast/visit.rs b/src/librustc_ast/visit.rs index ccab46703dffe..2c3d1e97df975 100644 --- a/src/librustc_ast/visit.rs +++ b/src/librustc_ast/visit.rs @@ -880,7 +880,7 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) { pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) { match attr.kind { AttrKind::Normal(ref item) => walk_mac_args(visitor, &item.args), - AttrKind::DocComment(_) => {} + AttrKind::DocComment(..) => {} } } diff --git a/src/librustc_ast_lowering/lib.rs b/src/librustc_ast_lowering/lib.rs index 9df7ad2a9acf4..077a07c1bfa15 100644 --- a/src/librustc_ast_lowering/lib.rs +++ b/src/librustc_ast_lowering/lib.rs @@ -981,7 +981,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { path: item.path.clone(), args: self.lower_mac_args(&item.args), }), - AttrKind::DocComment(comment) => AttrKind::DocComment(comment), + AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data), }; Attribute { kind, id: attr.id, style: attr.style, span: attr.span } diff --git a/src/librustc_ast_pretty/pprust.rs b/src/librustc_ast_pretty/pprust.rs index 4b228629ad719..243cd24e306e5 100644 --- a/src/librustc_ast_pretty/pprust.rs +++ b/src/librustc_ast_pretty/pprust.rs @@ -8,7 +8,7 @@ use rustc_ast::ast::{InlineAsmOperand, InlineAsmRegOrRegClass}; use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_ast::attr; use rustc_ast::ptr::P; -use rustc_ast::token::{self, BinOpToken, DelimToken, Nonterminal, Token, TokenKind}; +use rustc_ast::token::{self, BinOpToken, CommentKind, DelimToken, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{TokenStream, TokenTree}; use rustc_ast::util::parser::{self, AssocOp, Fixity}; use rustc_ast::util::{classify, comments}; @@ -152,8 +152,8 @@ pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String { // and also addresses some specific regressions described in #63896 and #73345. fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { if let TokenTree::Token(token) = prev { - if let token::DocComment(s) = token.kind { - return !s.as_str().starts_with("//"); + if let token::DocComment(comment_kind, ..) = token.kind { + return comment_kind != CommentKind::Line; } } match tt { @@ -194,6 +194,19 @@ fn binop_to_string(op: BinOpToken) -> &'static str { } } +fn doc_comment_to_string( + comment_kind: CommentKind, + attr_style: ast::AttrStyle, + data: Symbol, +) -> String { + match (comment_kind, attr_style) { + (CommentKind::Line, ast::AttrStyle::Outer) => format!("///{}", data), + (CommentKind::Line, ast::AttrStyle::Inner) => format!("//!{}", data), + (CommentKind::Block, ast::AttrStyle::Outer) => format!("/**{}*/", data), + (CommentKind::Block, ast::AttrStyle::Inner) => format!("/*!{}*/", data), + } +} + pub fn literal_to_string(lit: token::Lit) -> String { let token::Lit { kind, symbol, suffix } = lit; let mut out = match kind { @@ -271,7 +284,9 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option) token::Lifetime(s) => s.to_string(), /* Other */ - token::DocComment(s) => s.to_string(), + token::DocComment(comment_kind, attr_style, data) => { + doc_comment_to_string(comment_kind, attr_style, data) + } token::Eof => "".to_string(), token::Whitespace => " ".to_string(), token::Comment => "/* */".to_string(), @@ -599,8 +614,8 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere self.print_attr_item(&item, attr.span); self.word("]"); } - ast::AttrKind::DocComment(comment) => { - self.word(comment.to_string()); + ast::AttrKind::DocComment(comment_kind, data) => { + self.word(doc_comment_to_string(comment_kind, attr.style, data)); self.hardbreak() } } diff --git a/src/librustc_expand/parse/lexer/tests.rs b/src/librustc_expand/parse/lexer/tests.rs index b3775c78e7345..0b51abf385fd9 100644 --- a/src/librustc_expand/parse/lexer/tests.rs +++ b/src/librustc_expand/parse/lexer/tests.rs @@ -1,5 +1,5 @@ -use rustc_ast::token::{self, Token, TokenKind}; -use rustc_ast::util::comments::is_doc_comment; +use rustc_ast::ast::AttrStyle; +use rustc_ast::token::{self, CommentKind, Token, TokenKind}; use rustc_ast::with_default_session_globals; use rustc_data_structures::sync::Lrc; use rustc_errors::{emitter::EmitterWriter, Handler}; @@ -223,13 +223,6 @@ fn literal_suffixes() { }) } -#[test] -fn line_doc_comments() { - assert!(is_doc_comment("///")); - assert!(is_doc_comment("/// blah")); - assert!(!is_doc_comment("////")); -} - #[test] fn nested_block_comments() { with_default_session_globals(|| { @@ -251,6 +244,9 @@ fn crlf_comments() { assert_eq!(comment.kind, token::Comment); assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7))); assert_eq!(lexer.next_token(), token::Whitespace); - assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test"))); + assert_eq!( + lexer.next_token(), + token::DocComment(CommentKind::Line, AttrStyle::Outer, Symbol::intern(" test")) + ); }) } diff --git a/src/librustc_expand/parse/tests.rs b/src/librustc_expand/parse/tests.rs index fc9b9f2dab04e..d6301c8a82ee8 100644 --- a/src/librustc_expand/parse/tests.rs +++ b/src/librustc_expand/parse/tests.rs @@ -244,20 +244,20 @@ fn crlf_doc_comments() { let source = "/// doc comment\r\nfn foo() {}".to_string(); let item = parse_item_from_source_str(name_1, source, &sess).unwrap().unwrap(); let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); - assert_eq!(doc.as_str(), "/// doc comment"); + assert_eq!(doc.as_str(), " doc comment"); let name_2 = FileName::Custom("crlf_source_2".to_string()); let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string(); let item = parse_item_from_source_str(name_2, source, &sess).unwrap().unwrap(); let docs = item.attrs.iter().filter_map(|at| at.doc_str()).collect::>(); - let b: &[_] = &[Symbol::intern("/// doc comment"), Symbol::intern("/// line 2")]; + let b: &[_] = &[Symbol::intern(" doc comment"), Symbol::intern(" line 2")]; assert_eq!(&docs[..], b); let name_3 = FileName::Custom("clrf_source_3".to_string()); let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string(); let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap(); let doc = item.attrs.iter().filter_map(|at| at.doc_str()).next().unwrap(); - assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */"); + assert_eq!(doc.as_str(), " doc comment\n * with CRLF "); }); } diff --git a/src/librustc_expand/proc_macro_server.rs b/src/librustc_expand/proc_macro_server.rs index 881d7b84b70b0..d37742f2e43a3 100644 --- a/src/librustc_expand/proc_macro_server.rs +++ b/src/librustc_expand/proc_macro_server.rs @@ -148,9 +148,8 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> tt!(Punct::new('\'', true)) } Literal(lit) => tt!(Literal { lit }), - DocComment(c) => { - let style = comments::doc_comment_style(c); - let stripped = comments::strip_doc_comment_decoration(c); + DocComment(comment_kind, attr_style, data) => { + let stripped = comments::strip_doc_comment_decoration(data, comment_kind); let mut escaped = String::new(); for ch in stripped.chars() { escaped.extend(ch.escape_debug()); @@ -169,7 +168,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> span: DelimSpan::from_single(span), flatten: false, })); - if style == ast::AttrStyle::Inner { + if attr_style == ast::AttrStyle::Inner { stack.push(tt!(Punct::new('!', false))); } tt!(Punct::new('#', false)) diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs index 2b0e637c74e5a..5feb58dfa01c1 100644 --- a/src/librustc_parse/lexer/mod.rs +++ b/src/librustc_parse/lexer/mod.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{self, Token, TokenKind}; +use rustc_ast::token::{self, CommentKind, Token, TokenKind}; use rustc_ast::util::comments; use rustc_data_structures::sync::Lrc; use rustc_errors::{error_code, Applicability, DiagnosticBuilder, FatalError}; @@ -170,22 +170,19 @@ impl<'a> StringReader<'a> { match token { rustc_lexer::TokenKind::LineComment => { let string = self.str_from(start); - // comments with only more "/"s are not doc comments - if comments::is_line_doc_comment(string) { + if let Some(attr_style) = comments::line_doc_comment_style(string) { self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment"); - token::DocComment(Symbol::intern(string)) + token::DocComment(CommentKind::Line, attr_style, Symbol::intern(&string[3..])) } else { token::Comment } } rustc_lexer::TokenKind::BlockComment { terminated } => { let string = self.str_from(start); - // block comments starting with "/**" or "/*!" are doc-comments - // but comments with only "*"s between two "/"s are not - let is_doc_comment = comments::is_block_doc_comment(string); + let attr_style = comments::block_doc_comment_style(string, terminated); if !terminated { - let msg = if is_doc_comment { + let msg = if attr_style.is_some() { "unterminated block doc-comment" } else { "unterminated block comment" @@ -202,9 +199,13 @@ impl<'a> StringReader<'a> { FatalError.raise(); } - if is_doc_comment { + if let Some(attr_style) = attr_style { self.forbid_bare_cr(start, string, "bare CR not allowed in block doc-comment"); - token::DocComment(Symbol::intern(string)) + token::DocComment( + CommentKind::Block, + attr_style, + Symbol::intern(&string[3..string.len() - if terminated { 2 } else { 0 }]), + ) } else { token::Comment } diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs index 3319ca44da467..723e4333790ae 100644 --- a/src/librustc_parse/lib.rs +++ b/src/librustc_parse/lib.rs @@ -486,7 +486,9 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool { (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b, - (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, + (&DocComment(a1, a2, a3), &DocComment(b1, b2, b3)) => a1 == b1 && a2 == b2 && a3 == b3, + + (&Shebang(a), &Shebang(b)) => a == b, (&Literal(a), &Literal(b)) => a == b, @@ -524,7 +526,7 @@ fn prepend_attrs( let item = match attr.kind { ast::AttrKind::Normal(ref item) => item, - ast::AttrKind::DocComment(_) => { + ast::AttrKind::DocComment(..) => { let stream = parse_stream_from_source_str(macro_filename, source, sess, Some(span)); builder.push(stream); continue; diff --git a/src/librustc_parse/parser/attr.rs b/src/librustc_parse/parser/attr.rs index 8b67f4743c6b6..b6a8ee71beb0c 100644 --- a/src/librustc_parse/parser/attr.rs +++ b/src/librustc_parse/parser/attr.rs @@ -2,10 +2,9 @@ use super::{Parser, PathStyle}; use rustc_ast::ast; use rustc_ast::attr; use rustc_ast::token::{self, Nonterminal}; -use rustc_ast::util::comments; use rustc_ast_pretty::pprust; use rustc_errors::{error_code, PResult}; -use rustc_span::{Span, Symbol}; +use rustc_span::Span; use log::debug; @@ -47,8 +46,8 @@ impl<'a> Parser<'a> { let attr = self.parse_attribute_with_inner_parse_policy(inner_parse_policy)?; attrs.push(attr); just_parsed_doc_comment = false; - } else if let token::DocComment(s) = self.token.kind { - let attr = self.mk_doc_comment(s); + } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { + let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); if attr.style != ast::AttrStyle::Outer { self.sess .span_diagnostic @@ -73,10 +72,6 @@ impl<'a> Parser<'a> { Ok(attrs) } - fn mk_doc_comment(&self, s: Symbol) -> ast::Attribute { - attr::mk_doc_comment(comments::doc_comment_style(s), s, self.token.span) - } - /// Matches `attribute = # ! [ meta_item ]`. /// /// If `permit_inner` is `true`, then a leading `!` indicates an inner @@ -184,9 +179,9 @@ impl<'a> Parser<'a> { let attr = self.parse_attribute(true)?; assert_eq!(attr.style, ast::AttrStyle::Inner); attrs.push(attr); - } else if let token::DocComment(s) = self.token.kind { + } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { // We need to get the position of this token before we bump. - let attr = self.mk_doc_comment(s); + let attr = attr::mk_doc_comment(comment_kind, attr_style, data, self.token.span); if attr.style == ast::AttrStyle::Inner { attrs.push(attr); self.bump(); diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs index 5e9411327cabd..2854356ab0fc6 100644 --- a/src/librustc_parse/parser/diagnostics.rs +++ b/src/librustc_parse/parser/diagnostics.rs @@ -1419,7 +1419,7 @@ impl<'a> Parser<'a> { } pub(super) fn eat_incorrect_doc_comment_for_param_type(&mut self) { - if let token::DocComment(_) = self.token.kind { + if let token::DocComment(..) = self.token.kind { self.struct_span_err( self.token.span, "documentation comments cannot be applied to a function parameter's type", diff --git a/src/librustc_parse/parser/item.rs b/src/librustc_parse/parser/item.rs index 5923a185dcf93..10d214e52abdb 100644 --- a/src/librustc_parse/parser/item.rs +++ b/src/librustc_parse/parser/item.rs @@ -610,7 +610,7 @@ impl<'a> Parser<'a> { /// Recover on a doc comment before `}`. fn recover_doc_comment_before_brace(&mut self) -> bool { - if let token::DocComment(_) = self.token.kind { + if let token::DocComment(..) = self.token.kind { if self.look_ahead(1, |tok| tok == &token::CloseDelim(token::Brace)) { struct_span_err!( self.diagnostic(), @@ -1231,7 +1231,7 @@ impl<'a> Parser<'a> { self.bump(); } token::CloseDelim(token::Brace) => {} - token::DocComment(_) => { + token::DocComment(..) => { let previous_span = self.prev_token.span; let mut err = self.span_fatal_err(self.token.span, Error::UselessDocComment); self.bump(); // consume the doc comment diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 2509a9792215d..1b7fc4dafa2d4 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -22,7 +22,7 @@ use rustc_ast::ast::{ use rustc_ast::ptr::P; use rustc_ast::token::{self, DelimToken, Token, TokenKind}; use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndJoint}; -use rustc_ast::util::comments::{doc_comment_style, strip_doc_comment_decoration}; +use rustc_ast::util::comments::strip_doc_comment_decoration; use rustc_ast_pretty::pprust; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult}; use rustc_session::parse::ParseSess; @@ -209,12 +209,14 @@ impl TokenCursor { } fn next_desugared(&mut self) -> Token { - let (name, sp) = match self.next() { - Token { kind: token::DocComment(name), span } => (name, span), + let (data, comment_kind, attr_style, sp) = match self.next() { + Token { kind: token::DocComment(comment_kind, attr_style, data), span } => { + (data, comment_kind, attr_style, span) + } tok => return tok, }; - let stripped = strip_doc_comment_decoration(name); + let stripped = strip_doc_comment_decoration(data, comment_kind); // Searches for the occurrences of `"#*` and returns the minimum number of `#`s // required to wrap the text. @@ -251,7 +253,7 @@ impl TokenCursor { TokenCursorFrame::new( delim_span, token::NoDelim, - &if doc_comment_style(name) == AttrStyle::Inner { + &if attr_style == AttrStyle::Inner { [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] .iter() .cloned() diff --git a/src/librustc_save_analysis/lib.rs b/src/librustc_save_analysis/lib.rs index 8e379a3510038..e1e8dd0180765 100644 --- a/src/librustc_save_analysis/lib.rs +++ b/src/librustc_save_analysis/lib.rs @@ -822,8 +822,8 @@ impl<'tcx> SaveContext<'tcx> { for attr in attrs { if let Some(val) = attr.doc_str() { - if attr.is_doc_comment() { - result.push_str(&strip_doc_comment_decoration(val)); + if let ast::AttrKind::DocComment(comment_kind, _) = attr.kind { + result.push_str(&strip_doc_comment_decoration(val, comment_kind)); } else { result.push_str(&val.as_str()); } diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 1bea41b658532..f0f890e06dd85 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -506,11 +506,15 @@ impl Attributes { .iter() .filter_map(|attr| { if let Some(value) = attr.doc_str() { - let (value, mk_fragment): (_, fn(_, _, _) -> _) = if attr.is_doc_comment() { - (strip_doc_comment_decoration(value), DocFragment::SugaredDoc) - } else { - (value.to_string(), DocFragment::RawDoc) - }; + let (value, mk_fragment): (_, fn(_, _, _) -> _) = + if let ast::AttrKind::DocComment(comment_kind, _) = attr.kind { + ( + strip_doc_comment_decoration(value, comment_kind), + DocFragment::SugaredDoc, + ) + } else { + (value.to_string(), DocFragment::RawDoc) + }; let line = doc_line; doc_line += value.lines().count(); From 000c070b70eba07a903db82fb8505433842fe1f1 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 22 Jul 2020 11:42:36 +0300 Subject: [PATCH 2/5] rustc_ast/comments: Modernize some enum reexports --- src/librustc_ast/util/comments.rs | 18 ++++++++++-------- src/librustc_ast_pretty/pprust.rs | 25 +++++++++++++------------ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/librustc_ast/util/comments.rs b/src/librustc_ast/util/comments.rs index 543c22f4def05..20d44d5a6a739 100644 --- a/src/librustc_ast/util/comments.rs +++ b/src/librustc_ast/util/comments.rs @@ -1,5 +1,3 @@ -pub use CommentStyle::*; - use crate::ast::AttrStyle; use crate::token::CommentKind; use rustc_span::source_map::SourceMap; @@ -198,7 +196,7 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec Vec Vec true, }; let style = match (code_to_the_left, code_to_the_right) { - (_, true) => Mixed, - (false, false) => Isolated, - (true, false) => Trailing, + (_, true) => CommentStyle::Mixed, + (false, false) => CommentStyle::Isolated, + (true, false) => CommentStyle::Trailing, }; // Count the number of chars since the start of the line by rescanning. @@ -246,7 +244,11 @@ pub fn gather_comments(sm: &SourceMap, path: FileName, src: String) -> Vec { if line_doc_comment_style(token_text).is_none() { comments.push(Comment { - style: if code_to_the_left { Trailing } else { Isolated }, + style: if code_to_the_left { + CommentStyle::Trailing + } else { + CommentStyle::Isolated + }, lines: vec![token_text.to_string()], pos: start_bpos + BytePos(pos as u32), }) diff --git a/src/librustc_ast_pretty/pprust.rs b/src/librustc_ast_pretty/pprust.rs index 243cd24e306e5..9d9ca78de5588 100644 --- a/src/librustc_ast_pretty/pprust.rs +++ b/src/librustc_ast_pretty/pprust.rs @@ -10,8 +10,9 @@ use rustc_ast::attr; use rustc_ast::ptr::P; use rustc_ast::token::{self, BinOpToken, CommentKind, DelimToken, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::{TokenStream, TokenTree}; +use rustc_ast::util::classify; +use rustc_ast::util::comments::{gather_comments, Comment, CommentStyle}; use rustc_ast::util::parser::{self, AssocOp, Fixity}; -use rustc_ast::util::{classify, comments}; use rustc_span::edition::Edition; use rustc_span::source_map::{SourceMap, Spanned}; use rustc_span::symbol::{kw, sym, Ident, IdentPrinter, Symbol}; @@ -50,17 +51,17 @@ impl PpAnn for NoAnn {} pub struct Comments<'a> { sm: &'a SourceMap, - comments: Vec, + comments: Vec, current: usize, } impl<'a> Comments<'a> { pub fn new(sm: &'a SourceMap, filename: FileName, input: String) -> Comments<'a> { - let comments = comments::gather_comments(sm, filename, input); + let comments = gather_comments(sm, filename, input); Comments { sm, comments, current: 0 } } - pub fn next(&self) -> Option { + pub fn next(&self) -> Option { self.comments.get(self.current).cloned() } @@ -68,9 +69,9 @@ impl<'a> Comments<'a> { &mut self, span: rustc_span::Span, next_pos: Option, - ) -> Option { + ) -> Option { if let Some(cmnt) = self.next() { - if cmnt.style != comments::Trailing { + if cmnt.style != CommentStyle::Trailing { return None; } let span_line = self.sm.lookup_char_pos(span.hi()); @@ -462,9 +463,9 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere } } - fn print_comment(&mut self, cmnt: &comments::Comment) { + fn print_comment(&mut self, cmnt: &Comment) { match cmnt.style { - comments::Mixed => { + CommentStyle::Mixed => { if !self.is_beginning_of_line() { self.zerobreak(); } @@ -483,7 +484,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere } self.zerobreak() } - comments::Isolated => { + CommentStyle::Isolated => { self.hardbreak_if_not_bol(); for line in &cmnt.lines { // Don't print empty lines because they will end up as trailing @@ -494,7 +495,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere self.hardbreak(); } } - comments::Trailing => { + CommentStyle::Trailing => { if !self.is_beginning_of_line() { self.word(" "); } @@ -512,7 +513,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere self.end(); } } - comments::BlankLine => { + CommentStyle::BlankLine => { // We need to do at least one, possibly two hardbreaks. let twice = match self.last_token() { pp::Token::String(s) => ";" == s, @@ -531,7 +532,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere } } - fn next_comment(&mut self) -> Option { + fn next_comment(&mut self) -> Option { self.comments().as_mut().and_then(|c| c.next()) } From 712de2b5211ec9aad434358d459e843ca9a4622d Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 22 Jul 2020 12:28:17 +0300 Subject: [PATCH 3/5] rustc_expand: Don not beautify doc comments before passing them to macros Beautify all doc strings in rustdoc instead, including those in `#[doc]` attributes --- src/librustc_ast/util/comments.rs | 25 +++++++++++------------- src/librustc_ast/util/comments/tests.rs | 18 ++++++++--------- src/librustc_expand/proc_macro_server.rs | 6 ++---- src/librustc_parse/parser/mod.rs | 16 +++++---------- src/librustc_save_analysis/lib.rs | 9 +++------ src/librustdoc/clean/types.rs | 17 +++++++--------- 6 files changed, 37 insertions(+), 54 deletions(-) diff --git a/src/librustc_ast/util/comments.rs b/src/librustc_ast/util/comments.rs index 20d44d5a6a739..a73891db160de 100644 --- a/src/librustc_ast/util/comments.rs +++ b/src/librustc_ast/util/comments.rs @@ -1,5 +1,4 @@ use crate::ast::AttrStyle; -use crate::token::CommentKind; use rustc_span::source_map::SourceMap; use rustc_span::{BytePos, CharPos, FileName, Pos, Symbol}; @@ -64,7 +63,9 @@ pub fn block_doc_comment_style(block_comment: &str, terminated: bool) -> Option< } } -pub fn strip_doc_comment_decoration(data: Symbol, comment_kind: CommentKind) -> String { +/// Makes a doc string more presentable to users. +/// Used by rustdoc and perhaps other tools, but not by rustc. +pub fn beautify_doc_string(data: Symbol) -> String { /// remove whitespace-only lines from the start/end of lines fn vertical_trim(lines: Vec) -> Vec { let mut i = 0; @@ -126,18 +127,14 @@ pub fn strip_doc_comment_decoration(data: Symbol, comment_kind: CommentKind) -> } } - match comment_kind { - CommentKind::Line => { - let data = data.as_str(); - let prefix_len = if data.starts_with('!') { 1 } else { 0 }; - data[prefix_len..].to_string() - } - CommentKind::Block => { - let lines = data.as_str().lines().map(|s| s.to_string()).collect::>(); - let lines = vertical_trim(lines); - let lines = horizontal_trim(lines); - lines.join("\n") - } + let data = data.as_str(); + if data.contains('\n') { + let lines = data.lines().map(|s| s.to_string()).collect::>(); + let lines = vertical_trim(lines); + let lines = horizontal_trim(lines); + lines.join("\n") + } else { + data.to_string() } } diff --git a/src/librustc_ast/util/comments/tests.rs b/src/librustc_ast/util/comments/tests.rs index 274e9ed39bb28..e95365d833742 100644 --- a/src/librustc_ast/util/comments/tests.rs +++ b/src/librustc_ast/util/comments/tests.rs @@ -12,7 +12,7 @@ fn line_doc_comments() { fn test_block_doc_comment_1() { with_default_session_globals(|| { let comment = "\n * Test \n ** Test\n * Test\n"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); + let stripped = beautify_doc_string(Symbol::intern(comment)); assert_eq!(stripped, " Test \n* Test\n Test"); }) } @@ -21,7 +21,7 @@ fn test_block_doc_comment_1() { fn test_block_doc_comment_2() { with_default_session_globals(|| { let comment = "\n * Test\n * Test\n"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); + let stripped = beautify_doc_string(Symbol::intern(comment)); assert_eq!(stripped, " Test\n Test"); }) } @@ -30,7 +30,7 @@ fn test_block_doc_comment_2() { fn test_block_doc_comment_3() { with_default_session_globals(|| { let comment = "\n let a: *i32;\n *a = 5;\n"; - let stripped = strip_doc_comment_decoration(Symbol::intern(comment), CommentKind::Block); + let stripped = beautify_doc_string(Symbol::intern(comment)); assert_eq!(stripped, " let a: *i32;\n *a = 5;"); }) } @@ -38,13 +38,13 @@ fn test_block_doc_comment_3() { #[test] fn test_line_doc_comment() { with_default_session_globals(|| { - let stripped = strip_doc_comment_decoration(Symbol::intern(" test"), CommentKind::Line); + let stripped = beautify_doc_string(Symbol::intern(" test")); assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("! test"), CommentKind::Line); - assert_eq!(stripped, " test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("test"), CommentKind::Line); - assert_eq!(stripped, "test"); - let stripped = strip_doc_comment_decoration(Symbol::intern("!test"), CommentKind::Line); + let stripped = beautify_doc_string(Symbol::intern("! test")); + assert_eq!(stripped, "! test"); + let stripped = beautify_doc_string(Symbol::intern("test")); assert_eq!(stripped, "test"); + let stripped = beautify_doc_string(Symbol::intern("!test")); + assert_eq!(stripped, "!test"); }) } diff --git a/src/librustc_expand/proc_macro_server.rs b/src/librustc_expand/proc_macro_server.rs index d37742f2e43a3..005db35da7a32 100644 --- a/src/librustc_expand/proc_macro_server.rs +++ b/src/librustc_expand/proc_macro_server.rs @@ -3,7 +3,6 @@ use crate::base::ExtCtxt; use rustc_ast::ast; use rustc_ast::token; use rustc_ast::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; -use rustc_ast::util::comments; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; use rustc_errors::Diagnostic; @@ -148,10 +147,9 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> tt!(Punct::new('\'', true)) } Literal(lit) => tt!(Literal { lit }), - DocComment(comment_kind, attr_style, data) => { - let stripped = comments::strip_doc_comment_decoration(data, comment_kind); + DocComment(_, attr_style, data) => { let mut escaped = String::new(); - for ch in stripped.chars() { + for ch in data.as_str().chars() { escaped.extend(ch.escape_debug()); } let stream = vec![ diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 1b7fc4dafa2d4..ededfc43669da 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -22,7 +22,6 @@ use rustc_ast::ast::{ use rustc_ast::ptr::P; use rustc_ast::token::{self, DelimToken, Token, TokenKind}; use rustc_ast::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndJoint}; -use rustc_ast::util::comments::strip_doc_comment_decoration; use rustc_ast_pretty::pprust; use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, FatalError, PResult}; use rustc_session::parse::ParseSess; @@ -209,20 +208,18 @@ impl TokenCursor { } fn next_desugared(&mut self) -> Token { - let (data, comment_kind, attr_style, sp) = match self.next() { - Token { kind: token::DocComment(comment_kind, attr_style, data), span } => { - (data, comment_kind, attr_style, span) + let (data, attr_style, sp) = match self.next() { + Token { kind: token::DocComment(_, attr_style, data), span } => { + (data, attr_style, span) } tok => return tok, }; - let stripped = strip_doc_comment_decoration(data, comment_kind); - // Searches for the occurrences of `"#*` and returns the minimum number of `#`s // required to wrap the text. let mut num_of_hashes = 0; let mut count = 0; - for ch in stripped.chars() { + for ch in data.as_str().chars() { count = match ch { '"' => 1, '#' if count > 0 => count + 1, @@ -238,10 +235,7 @@ impl TokenCursor { [ TokenTree::token(token::Ident(sym::doc, false), sp), TokenTree::token(token::Eq, sp), - TokenTree::token( - TokenKind::lit(token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None), - sp, - ), + TokenTree::token(TokenKind::lit(token::StrRaw(num_of_hashes), data, None), sp), ] .iter() .cloned() diff --git a/src/librustc_save_analysis/lib.rs b/src/librustc_save_analysis/lib.rs index e1e8dd0180765..d854835a02475 100644 --- a/src/librustc_save_analysis/lib.rs +++ b/src/librustc_save_analysis/lib.rs @@ -10,7 +10,7 @@ mod span_utils; mod sig; use rustc_ast::ast::{self}; -use rustc_ast::util::comments::strip_doc_comment_decoration; +use rustc_ast::util::comments::beautify_doc_string; use rustc_ast_pretty::pprust::attribute_to_string; use rustc_hir as hir; use rustc_hir::def::{DefKind as HirDefKind, Res}; @@ -822,11 +822,8 @@ impl<'tcx> SaveContext<'tcx> { for attr in attrs { if let Some(val) = attr.doc_str() { - if let ast::AttrKind::DocComment(comment_kind, _) = attr.kind { - result.push_str(&strip_doc_comment_decoration(val, comment_kind)); - } else { - result.push_str(&val.as_str()); - } + // FIXME: Should save-analysis beautify doc strings itself or leave it to users? + result.push_str(&beautify_doc_string(val)); result.push('\n'); } else if attr.check_name(sym::doc) { if let Some(meta_list) = attr.meta_item_list() { diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index f0f890e06dd85..44dd329d9ced3 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -10,7 +10,7 @@ use std::{slice, vec}; use rustc_ast::ast::{self, AttrStyle}; use rustc_ast::attr; -use rustc_ast::util::comments::strip_doc_comment_decoration; +use rustc_ast::util::comments::beautify_doc_string; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir as hir; use rustc_hir::def::Res; @@ -506,15 +506,12 @@ impl Attributes { .iter() .filter_map(|attr| { if let Some(value) = attr.doc_str() { - let (value, mk_fragment): (_, fn(_, _, _) -> _) = - if let ast::AttrKind::DocComment(comment_kind, _) = attr.kind { - ( - strip_doc_comment_decoration(value, comment_kind), - DocFragment::SugaredDoc, - ) - } else { - (value.to_string(), DocFragment::RawDoc) - }; + let value = beautify_doc_string(value); + let mk_fragment: fn(_, _, _) -> _ = if attr.is_doc_comment() { + DocFragment::SugaredDoc + } else { + DocFragment::RawDoc + }; let line = doc_line; doc_line += value.lines().count(); From 6b25c50ed467396c367d41eca6b8248136140974 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 22 Jul 2020 17:59:17 +0300 Subject: [PATCH 4/5] Fix clippy --- src/tools/clippy/clippy_lints/src/doc.rs | 80 ++++++++----------- .../clippy_lints/src/tabs_in_doc_comments.rs | 6 +- .../clippy_lints/src/utils/ast_utils.rs | 2 +- 3 files changed, 39 insertions(+), 49 deletions(-) diff --git a/src/tools/clippy/clippy_lints/src/doc.rs b/src/tools/clippy/clippy_lints/src/doc.rs index e87c33d1b09dd..94371b7d23e34 100644 --- a/src/tools/clippy/clippy_lints/src/doc.rs +++ b/src/tools/clippy/clippy_lints/src/doc.rs @@ -2,6 +2,7 @@ use crate::utils::{implements_trait, is_entrypoint_fn, is_type_diagnostic_item, use if_chain::if_chain; use itertools::Itertools; use rustc_ast::ast::{AttrKind, Attribute}; +use rustc_ast::token::CommentKind; use rustc_data_structures::fx::FxHashSet; use rustc_hir as hir; use rustc_lint::{LateContext, LateLintPass}; @@ -249,7 +250,7 @@ fn lint_for_missing_headers<'tcx>( } } -/// Cleanup documentation decoration (`///` and such). +/// Cleanup documentation decoration. /// /// We can't use `rustc_ast::attr::AttributeMethods::with_desugared_doc` or /// `rustc_ast::parse::lexer::comments::strip_doc_comment_decoration` because we @@ -257,54 +258,44 @@ fn lint_for_missing_headers<'tcx>( /// the spans but this function is inspired from the later. #[allow(clippy::cast_possible_truncation)] #[must_use] -pub fn strip_doc_comment_decoration(comment: &str, span: Span) -> (String, Vec<(usize, Span)>) { +pub fn strip_doc_comment_decoration(doc: &str, comment_kind: CommentKind, span: Span) -> (String, Vec<(usize, Span)>) { // one-line comments lose their prefix - const ONELINERS: &[&str] = &["///!", "///", "//!", "//"]; - for prefix in ONELINERS { - if comment.starts_with(*prefix) { - let doc = &comment[prefix.len()..]; - let mut doc = doc.to_owned(); - doc.push('\n'); - return ( - doc.to_owned(), - vec![(doc.len(), span.with_lo(span.lo() + BytePos(prefix.len() as u32)))], - ); - } + if comment_kind == CommentKind::Line { + let mut doc = doc.to_owned(); + doc.push('\n'); + let len = doc.len(); + return (doc, vec![(len, span.with_lo(span.lo() + BytePos(3)))]); } - if comment.starts_with("/*") { - let doc = &comment[3..comment.len() - 2]; - let mut sizes = vec![]; - let mut contains_initial_stars = false; - for line in doc.lines() { - let offset = line.as_ptr() as usize - comment.as_ptr() as usize; - debug_assert_eq!(offset as u32 as usize, offset); - contains_initial_stars |= line.trim_start().starts_with('*'); - // +1 for the newline - sizes.push((line.len() + 1, span.with_lo(span.lo() + BytePos(offset as u32)))); - } - if !contains_initial_stars { - return (doc.to_string(), sizes); - } - // remove the initial '*'s if any - let mut no_stars = String::with_capacity(doc.len()); - for line in doc.lines() { - let mut chars = line.chars(); - while let Some(c) = chars.next() { - if c.is_whitespace() { - no_stars.push(c); - } else { - no_stars.push(if c == '*' { ' ' } else { c }); - break; - } + let mut sizes = vec![]; + let mut contains_initial_stars = false; + for line in doc.lines() { + let offset = line.as_ptr() as usize - doc.as_ptr() as usize; + debug_assert_eq!(offset as u32 as usize, offset); + contains_initial_stars |= line.trim_start().starts_with('*'); + // +1 for the newline + sizes.push((line.len() + 1, span.with_lo(span.lo() + BytePos(3 + offset as u32)))); + } + if !contains_initial_stars { + return (doc.to_string(), sizes); + } + // remove the initial '*'s if any + let mut no_stars = String::with_capacity(doc.len()); + for line in doc.lines() { + let mut chars = line.chars(); + while let Some(c) = chars.next() { + if c.is_whitespace() { + no_stars.push(c); + } else { + no_stars.push(if c == '*' { ' ' } else { c }); + break; } - no_stars.push_str(chars.as_str()); - no_stars.push('\n'); } - return (no_stars, sizes); + no_stars.push_str(chars.as_str()); + no_stars.push('\n'); } - panic!("not a doc-comment: {}", comment); + (no_stars, sizes) } #[derive(Copy, Clone)] @@ -318,9 +309,8 @@ fn check_attrs<'a>(cx: &LateContext<'_>, valid_idents: &FxHashSet, attrs let mut spans = vec![]; for attr in attrs { - if let AttrKind::DocComment(ref comment) = attr.kind { - let comment = comment.to_string(); - let (comment, current_spans) = strip_doc_comment_decoration(&comment, attr.span); + if let AttrKind::DocComment(comment_kind, comment) = attr.kind { + let (comment, current_spans) = strip_doc_comment_decoration(&comment.as_str(), comment_kind, attr.span); spans.extend_from_slice(¤t_spans); doc.push_str(&comment); } else if attr.has_name(sym!(doc)) { diff --git a/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs b/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs index 7b673e15b764a..d00114eed696a 100644 --- a/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs +++ b/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs @@ -60,13 +60,13 @@ declare_lint_pass!(TabsInDocComments => [TABS_IN_DOC_COMMENTS]); impl TabsInDocComments { fn warn_if_tabs_in_doc(cx: &EarlyContext<'_>, attr: &ast::Attribute) { - if let ast::AttrKind::DocComment(comment) = attr.kind { + if let ast::AttrKind::DocComment(_, comment) = attr.kind { let comment = comment.as_str(); for (lo, hi) in get_chunks_of_tabs(&comment) { let new_span = Span::new( - attr.span.lo() + BytePos(lo), - attr.span.lo() + BytePos(hi), + attr.span.lo() + BytePos(3 + lo), + attr.span.lo() + BytePos(3 + hi), attr.span.ctxt(), ); span_lint_and_sugg( diff --git a/src/tools/clippy/clippy_lints/src/utils/ast_utils.rs b/src/tools/clippy/clippy_lints/src/utils/ast_utils.rs index 58c1103da9f7d..ad02bc5fd8e7d 100755 --- a/src/tools/clippy/clippy_lints/src/utils/ast_utils.rs +++ b/src/tools/clippy/clippy_lints/src/utils/ast_utils.rs @@ -506,7 +506,7 @@ pub fn eq_attr(l: &Attribute, r: &Attribute) -> bool { use AttrKind::*; l.style == r.style && match (&l.kind, &r.kind) { - (DocComment(l), DocComment(r)) => l == r, + (DocComment(l1, l2), DocComment(r1, r2)) => l1 == r1 && l2 == r2, (Normal(l), Normal(r)) => eq_path(&l.path, &r.path) && eq_mac_args(&l.args, &r.args), _ => false, } From a7eabec1df53fe726455c8088ecc0da07dd4009d Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Wed, 5 Aug 2020 00:26:23 +0300 Subject: [PATCH 5/5] Add some comments for magic numbers + Add tests --- src/librustc_parse/lexer/mod.rs | 3 ++ .../ui/proc-macro/doc-comment-preserved.rs | 24 +++++++++ .../proc-macro/doc-comment-preserved.stdout | 54 +++++++++++++++++++ src/tools/clippy/clippy_lints/src/doc.rs | 3 +- .../clippy_lints/src/tabs_in_doc_comments.rs | 1 + 5 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/test/ui/proc-macro/doc-comment-preserved.rs create mode 100644 src/test/ui/proc-macro/doc-comment-preserved.stdout diff --git a/src/librustc_parse/lexer/mod.rs b/src/librustc_parse/lexer/mod.rs index 5feb58dfa01c1..c3a79660eb9b9 100644 --- a/src/librustc_parse/lexer/mod.rs +++ b/src/librustc_parse/lexer/mod.rs @@ -172,6 +172,7 @@ impl<'a> StringReader<'a> { let string = self.str_from(start); if let Some(attr_style) = comments::line_doc_comment_style(string) { self.forbid_bare_cr(start, string, "bare CR not allowed in doc-comment"); + // Opening delimiter of the length 3 is not included into the symbol. token::DocComment(CommentKind::Line, attr_style, Symbol::intern(&string[3..])) } else { token::Comment @@ -201,6 +202,8 @@ impl<'a> StringReader<'a> { if let Some(attr_style) = attr_style { self.forbid_bare_cr(start, string, "bare CR not allowed in block doc-comment"); + // Opening delimiter of the length 3 and closing delimiter of the length 2 + // are not included into the symbol. token::DocComment( CommentKind::Block, attr_style, diff --git a/src/test/ui/proc-macro/doc-comment-preserved.rs b/src/test/ui/proc-macro/doc-comment-preserved.rs new file mode 100644 index 0000000000000..c2724ae18066a --- /dev/null +++ b/src/test/ui/proc-macro/doc-comment-preserved.rs @@ -0,0 +1,24 @@ +// check-pass +// aux-build:test-macros.rs + +// Anonymize unstable non-dummy spans while still showing dummy spans `0..0`. +// normalize-stdout-test "bytes\([^0]\w*\.\.(\w+)\)" -> "bytes(LO..$1)" +// normalize-stdout-test "bytes\((\w+)\.\.[^0]\w*\)" -> "bytes($1..HI)" + +#[macro_use] +extern crate test_macros; + +print_bang! { + +/** +******* +* DOC * +* DOC * +* DOC * +******* +*/ +pub struct S; + +} + +fn main() {} diff --git a/src/test/ui/proc-macro/doc-comment-preserved.stdout b/src/test/ui/proc-macro/doc-comment-preserved.stdout new file mode 100644 index 0000000000000..f7904536a7613 --- /dev/null +++ b/src/test/ui/proc-macro/doc-comment-preserved.stdout @@ -0,0 +1,54 @@ +PRINT-BANG INPUT (DISPLAY): /** +******* +* DOC * +* DOC * +* DOC * +******* +*/ + pub struct S ; +PRINT-BANG RE-COLLECTED (DISPLAY): #[doc = "\n*******\n* DOC *\n* DOC *\n* DOC *\n*******\n"] pub struct S ; +PRINT-BANG INPUT (DEBUG): TokenStream [ + Punct { + ch: '#', + spacing: Alone, + span: #0 bytes(LO..HI), + }, + Group { + delimiter: Bracket, + stream: TokenStream [ + Ident { + ident: "doc", + span: #0 bytes(LO..HI), + }, + Punct { + ch: '=', + spacing: Alone, + span: #0 bytes(LO..HI), + }, + Literal { + kind: Str, + symbol: "\n*******\n* DOC *\n* DOC *\n* DOC *\n*******\n", + suffix: None, + span: #0 bytes(LO..HI), + }, + ], + span: #0 bytes(LO..HI), + }, + Ident { + ident: "pub", + span: #0 bytes(LO..HI), + }, + Ident { + ident: "struct", + span: #0 bytes(LO..HI), + }, + Ident { + ident: "S", + span: #0 bytes(LO..HI), + }, + Punct { + ch: ';', + spacing: Alone, + span: #0 bytes(LO..HI), + }, +] diff --git a/src/tools/clippy/clippy_lints/src/doc.rs b/src/tools/clippy/clippy_lints/src/doc.rs index 94371b7d23e34..6ce36fd2360e1 100644 --- a/src/tools/clippy/clippy_lints/src/doc.rs +++ b/src/tools/clippy/clippy_lints/src/doc.rs @@ -264,6 +264,7 @@ pub fn strip_doc_comment_decoration(doc: &str, comment_kind: CommentKind, span: let mut doc = doc.to_owned(); doc.push('\n'); let len = doc.len(); + // +3 skips the opening delimiter return (doc, vec![(len, span.with_lo(span.lo() + BytePos(3)))]); } @@ -273,7 +274,7 @@ pub fn strip_doc_comment_decoration(doc: &str, comment_kind: CommentKind, span: let offset = line.as_ptr() as usize - doc.as_ptr() as usize; debug_assert_eq!(offset as u32 as usize, offset); contains_initial_stars |= line.trim_start().starts_with('*'); - // +1 for the newline + // +1 adds the newline, +3 skips the opening delimiter sizes.push((line.len() + 1, span.with_lo(span.lo() + BytePos(3 + offset as u32)))); } if !contains_initial_stars { diff --git a/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs b/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs index d00114eed696a..74ccd9235de85 100644 --- a/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs +++ b/src/tools/clippy/clippy_lints/src/tabs_in_doc_comments.rs @@ -64,6 +64,7 @@ impl TabsInDocComments { let comment = comment.as_str(); for (lo, hi) in get_chunks_of_tabs(&comment) { + // +3 skips the opening delimiter let new_span = Span::new( attr.span.lo() + BytePos(3 + lo), attr.span.lo() + BytePos(3 + hi),