diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index a33181e5925cd..4d79deb5a3d3a 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -349,9 +349,18 @@ impl<'a> HashStable> for token::TokenKind { } } +impl<'a> HashStable> for token::IsJoint { + fn hash_stable(&self, + hcx: &mut StableHashingContext<'a>, + hasher: &mut StableHasher) { + mem::discriminant(self).hash_stable(hcx, hasher); + } +} + impl_stable_hash_for!(struct token::Token { kind, - span + span, + joint }); impl_stable_hash_for!(enum ::syntax::ast::NestedMetaItem { diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 9d06b926f972e..fad6a0bb4fe66 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -478,12 +478,12 @@ impl MetaItem { let mod_sep_span = Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt()); - idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); + idents.push(TokenTree::token(token::ModSep, mod_sep_span)); } - idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); + idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident))); last_pos = segment.ident.span.hi(); } - self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); + self.node.tokens(self.span).append_to(&mut idents); TokenStream::new(idents) } @@ -492,8 +492,8 @@ impl MetaItem { { // FIXME: Share code with `parse_path`. let path = match tokens.next() { - Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) | - Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: { + Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span, .. })) | + Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span, .. })) => 'arm: { let mut segments = if let token::Ident(name, _) = kind { if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() { @@ -506,7 +506,7 @@ impl MetaItem { vec![PathSegment::path_root(span)] }; loop { - if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) + if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span, .. })) = tokens.next() { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { @@ -547,17 +547,17 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::empty(), MetaItemKind::NameValue(ref lit) => { - let mut vec = vec![TokenTree::token(token::Eq, span).into()]; - lit.tokens().append_to_tree_and_joint_vec(&mut vec); + let mut vec = vec![TokenTree::token(token::Eq, span)]; + lit.tokens().append_to(&mut vec); TokenStream::new(vec) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); for (i, item) in list.iter().enumerate() { if i > 0 { - tokens.push(TokenTree::token(token::Comma, span).into()); + tokens.push(TokenTree::token(token::Comma, span)); } - item.tokens().append_to_tree_and_joint_vec(&mut tokens); + item.tokens().append_to(&mut tokens); } TokenTree::Delimited( DelimSpan::from_single(span), diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs index 3952e29a5f0d1..229c7725493cb 100644 --- a/src/libsyntax/ext/mbe/quoted.rs +++ b/src/libsyntax/ext/mbe/quoted.rs @@ -66,7 +66,7 @@ pub(super) fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if expect_matchers => { let span = match trees.next() { - Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { + Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span, .. })) => { match trees.next() { Some(tokenstream::TokenTree::Token(token)) => match token.ident() { Some((kind, _)) => { @@ -120,7 +120,9 @@ fn parse_tree( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { + tokenstream::TokenTree::Token(Token { + kind: token::Dollar, span, .. + }) => match trees.next() { // `tree` is followed by a delimited set of token trees. This indicates the beginning // of a repetition sequence in the macro (e.g. `$(pat)*`). Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs index ba818ebd35c7f..f30deed288b66 100644 --- a/src/libsyntax/ext/mbe/transcribe.rs +++ b/src/libsyntax/ext/mbe/transcribe.rs @@ -4,7 +4,7 @@ use crate::ext::mbe; use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use crate::mut_visit::{self, MutVisitor}; use crate::parse::token::{self, NtTT, Token}; -use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; use smallvec::{smallvec, SmallVec}; @@ -118,7 +118,7 @@ pub(super) fn transcribe( // // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level // again, and we are done transcribing. - let mut result: Vec = Vec::new(); + let mut result: Vec = Vec::new(); let mut result_stack = Vec::new(); let mut marker = Marker(cx.current_expansion.id, transparency); @@ -138,7 +138,7 @@ pub(super) fn transcribe( if repeat_idx < repeat_len { *idx = 0; if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone()).into()); + result.push(TokenTree::Token(sep.clone())); } continue; } @@ -241,11 +241,11 @@ pub(super) fn transcribe( // (e.g. `$x:tt`), but not when we are matching any other type of token // tree? if let NtTT(ref tt) = **nt { - result.push(tt.clone().into()); + result.push(tt.clone()); } else { marker.visit_span(&mut sp); let token = TokenTree::token(token::Interpolated(nt.clone()), sp); - result.push(token.into()); + result.push(token); } } else { // We were unable to descend far enough. This is an error. @@ -259,8 +259,8 @@ pub(super) fn transcribe( // with modified syntax context. (I believe this supports nested macros). marker.visit_span(&mut sp); marker.visit_ident(&mut ident); - result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); + result.push(TokenTree::token(token::Dollar, sp)); + result.push(TokenTree::Token(Token::from_ast_ident(ident))); } } @@ -280,7 +280,7 @@ pub(super) fn transcribe( mbe::TokenTree::Token(token) => { let mut tt = TokenTree::Token(token); marker.visit_tt(&mut tt); - result.push(tt.into()); + result.push(tt); } // There should be no meta-var declarations in the invocation of a macro. diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index dfec9ee28809a..424915959ba61 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -2,7 +2,7 @@ use crate::ast; use crate::ext::base::ExtCtxt; use crate::parse::{self, token, ParseSess}; use crate::parse::lexer::comments; -use crate::tokenstream::{self, DelimSpan, IsJoint::*, TokenStream, TreeAndJoint}; +use crate::tokenstream::{self, DelimSpan, TokenStream}; use errors::Diagnostic; use rustc_data_structures::sync::Lrc; @@ -44,15 +44,14 @@ impl ToInternal for Delimiter { } } -impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> +impl FromInternal<(tokenstream::TokenTree, &'_ ParseSess, &'_ mut Vec)> for TokenTree { - fn from_internal(((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec)) + fn from_internal((tree, sess, stack): (tokenstream::TokenTree, &ParseSess, &mut Vec)) -> Self { use crate::parse::token::*; - let joint = is_joint == Joint; - let Token { kind, span } = match tree { + let Token { kind, span, joint } = match tree { tokenstream::TokenTree::Delimited(span, delim, tts) => { let delimiter = Delimiter::from_internal(delim); return TokenTree::Group(Group { @@ -63,6 +62,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> } tokenstream::TokenTree::Token(token) => token, }; + let joint = joint == Joint; macro_rules! tt { ($ty:ident { $($field:ident $(: $value:expr)*),+ $(,)? }) => ( @@ -262,8 +262,11 @@ impl ToInternal for TokenTree { _ => unreachable!(), }; - let tree = tokenstream::TokenTree::token(kind, span); - TokenStream::new(vec![(tree, if joint { Joint } else { NonJoint })]) + let token = Token::new(kind, span) + .with_joint(if joint { Joint } else { NonJoint }); + let tree = tokenstream::TokenTree::Token(token); + + TokenStream::new(vec![(tree)]) } } @@ -440,7 +443,7 @@ impl server::TokenStreamIter for Rustc<'_> { ) -> Option> { loop { let tree = iter.stack.pop().or_else(|| { - let next = iter.cursor.next_with_joint()?; + let next = iter.cursor.next()?; Some(TokenTree::from_internal((next, self.sess, &mut iter.stack))) })?; // HACK: The condition "dummy span + group with empty delimiter" represents an AST diff --git a/src/libsyntax/mut_visit.rs b/src/libsyntax/mut_visit.rs index 5a37222ee5590..527c4fcf0f491 100644 --- a/src/libsyntax/mut_visit.rs +++ b/src/libsyntax/mut_visit.rs @@ -611,7 +611,7 @@ pub fn noop_visit_tt(tt: &mut TokenTree, vis: &mut T) { pub fn noop_visit_tts(TokenStream(tts): &mut TokenStream, vis: &mut T) { visit_opt(tts, |tts| { let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| vis.visit_tt(tree)); + visit_vec(tts, |tree| vis.visit_tt(tree)); }) } @@ -619,7 +619,7 @@ pub fn noop_visit_tts(TokenStream(tts): &mut TokenStream, vis: &m // In practice the ident part is not actually used by specific visitors right now, // but there's a test below checking that it works. pub fn noop_visit_token(t: &mut Token, vis: &mut T) { - let Token { kind, span } = t; + let Token { kind, span, .. } = t; match kind { token::Ident(name, _) | token::Lifetime(name) => { let mut ident = Ident::new(*name, *span); diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index e5ba7e45309dd..c61b54a188c05 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -2,9 +2,9 @@ use syntax_pos::Span; use crate::print::pprust::token_to_string; use crate::parse::lexer::{StringReader, UnmatchedBrace}; -use crate::parse::token::{self, Token}; +use crate::parse::token::{self, Token, IsJoint::{self, *}}; use crate::parse::PResult; -use crate::tokenstream::{DelimSpan, IsJoint::{self, *}, TokenStream, TokenTree, TreeAndJoint}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree}; impl<'a> StringReader<'a> { crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec) { @@ -67,7 +67,7 @@ impl<'a> TokenTreesReader<'a> { } } - fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> { + fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> { let sm = self.string_reader.sess.source_map(); match self.token.kind { token::Eof => { @@ -191,7 +191,7 @@ impl<'a> TokenTreesReader<'a> { delim_span, delim, tts.into() - ).into()) + )) }, token::CloseDelim(_) => { // An unexpected closing delimiter (i.e., there is no @@ -204,10 +204,10 @@ impl<'a> TokenTreesReader<'a> { Err(err) }, _ => { - let tt = TokenTree::Token(self.token.take()); + let token = self.token.take(); self.real_token(); - let is_joint = self.joint_to_prev == Joint && self.token.is_op(); - Ok((tt, if is_joint { Joint } else { NonJoint })) + let is_joint = self.joint_to_prev == Joint && token.is_op() && self.token.is_op(); + Ok(TokenTree::Token(token.with_joint(if is_joint { Joint } else { NonJoint }))) } } } @@ -231,21 +231,21 @@ impl<'a> TokenTreesReader<'a> { #[derive(Default)] struct TokenStreamBuilder { - buf: Vec, + buf: Vec, } impl TokenStreamBuilder { - fn push(&mut self, (tree, joint): TreeAndJoint) { - if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() { + fn push(&mut self, tree: TokenTree) { + if let Some(TokenTree::Token(prev_token)) = self.buf.last() { if let TokenTree::Token(token) = &tree { if let Some(glued) = prev_token.glue(token) { self.buf.pop(); - self.buf.push((TokenTree::Token(glued), joint)); + self.buf.push(TokenTree::Token(glued)); return; } } } - self.buf.push((tree, joint)) + self.buf.push(tree) } fn into_token_stream(self) -> TokenStream { diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index b2b6504919e96..890c2b488cd84 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -24,7 +24,7 @@ use crate::print::pprust; use crate::ptr::P; use crate::source_map::{self, respan}; use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint}; +use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream}; use crate::ThinVec; use errors::{Applicability, DiagnosticId, FatalError}; @@ -195,8 +195,8 @@ struct TokenCursorFrame { /// on the parser. #[derive(Clone)] crate enum LastToken { - Collecting(Vec), - Was(Option), + Collecting(Vec), + Was(Option), } impl TokenCursorFrame { @@ -231,8 +231,8 @@ impl TokenCursor { }; match self.frame.last_token { - LastToken::Collecting(ref mut v) => v.push(tree.clone().into()), - LastToken::Was(ref mut t) => *t = Some(tree.clone().into()), + LastToken::Collecting(ref mut v) => v.push(tree.clone()), + LastToken::Was(ref mut t) => *t = Some(tree.clone()), } match tree { @@ -247,7 +247,7 @@ impl TokenCursor { fn next_desugared(&mut self) -> Token { let (name, sp) = match self.next() { - Token { kind: token::DocComment(name), span } => (name, span), + Token { kind: token::DocComment(name), span, .. } => (name, span), tok => return tok, }; @@ -1173,7 +1173,7 @@ impl<'a> Parser<'a> { loop { match self.token.kind { token::Eof | token::CloseDelim(..) => break, - _ => result.push(self.parse_token_tree().into()), + _ => result.push(self.parse_token_tree()), } } TokenStream::new(result) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index fe3b51aa246b8..3e8ae80ec645d 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -265,10 +265,24 @@ pub enum TokenKind { #[cfg(target_arch = "x86_64")] static_assert_size!(TokenKind, 16); +#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)] +pub enum IsJoint { + Joint, + NonJoint +} +pub use IsJoint::*; + #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] pub struct Token { pub kind: TokenKind, pub span: Span, + /// Is this operator token immediately followed by another operator token, + /// without whitespace? + /// + /// At the moment, this field is only used for proc_macros, and joint tokens + /// are usually represented by dedicated token kinds. We want to transition + /// to using jointness everywhere though (#63689). + pub joint: IsJoint } impl TokenKind { @@ -289,7 +303,11 @@ impl TokenKind { impl Token { crate fn new(kind: TokenKind, span: Span) -> Self { - Token { kind, span } + Token { kind, span, joint: NonJoint } + } + + crate fn with_joint(self, joint: IsJoint) -> Self { + Token { joint, ..self } } /// Some token that will be thrown away later. @@ -549,6 +567,9 @@ impl Token { } crate fn glue(&self, joint: &Token) -> Option { + if self.joint == NonJoint { + return None; + } let kind = match self.kind { Eq => match joint.kind { Eq => EqEq, @@ -604,8 +625,9 @@ impl Token { Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, }; - - Some(Token::new(kind, self.span.to(joint.span))) + let mut token = Token::new(kind, self.span.to(joint.span)); + token.joint = joint.joint; + Some(token) } // See comments in `Nonterminal::to_tokenstream` for why we care about diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 26cae2a8e7c42..b187bcce935c2 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -103,10 +103,6 @@ impl TokenTree { } } - pub fn joint(self) -> TokenStream { - TokenStream::new(vec![(self, Joint)]) - } - pub fn token(kind: TokenKind, span: Span) -> TokenTree { TokenTree::Token(Token::new(kind, span)) } @@ -143,22 +139,12 @@ impl TokenTree { /// empty stream is represented with `None`; it may be represented as a `Some` /// around an empty `Vec`. #[derive(Clone, Debug)] -pub struct TokenStream(pub Option>>); - -pub type TreeAndJoint = (TokenTree, IsJoint); +pub struct TokenStream(pub Option>>); // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] static_assert_size!(TokenStream, 8); -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum IsJoint { - Joint, - NonJoint -} - -use IsJoint::*; - impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. @@ -170,18 +156,18 @@ impl TokenStream { while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { - (_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, - ((TokenTree::Token(token_left), NonJoint), - (TokenTree::Token(token_right), _)) + (_, TokenTree::Token(Token { kind: token::Comma, .. })) => continue, + (TokenTree::Token(token_left), + TokenTree::Token(token_right)) if ((token_left.is_ident() && !token_left.is_reserved_ident()) || token_left.is_lit()) && ((token_right.is_ident() && !token_right.is_reserved_ident()) || token_right.is_lit()) => token_left.span, - ((TokenTree::Delimited(sp, ..), NonJoint), _) => sp.entire(), + (TokenTree::Delimited(sp, ..), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = (TokenTree::token(token::Comma, sp), NonJoint); + let comma = TokenTree::token(token::Comma, sp); suggestion = Some((pos, comma, sp)); } } @@ -200,13 +186,7 @@ impl TokenStream { impl From for TokenStream { fn from(tree: TokenTree) -> TokenStream { - TokenStream::new(vec![(tree, NonJoint)]) - } -} - -impl From for TreeAndJoint { - fn from(tree: TokenTree) -> TreeAndJoint { - (tree, NonJoint) + TokenStream::new(vec![tree]) } } @@ -267,14 +247,14 @@ impl TokenStream { } } - pub fn new(streams: Vec) -> TokenStream { + pub fn new(streams: Vec) -> TokenStream { match streams.len() { 0 => TokenStream(None), _ => TokenStream(Some(Lrc::new(streams))), } } - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec) { + pub fn append_to(self, vec: &mut Vec) { if let Some(stream) = self.0 { vec.extend(stream.iter().cloned()); } @@ -349,7 +329,7 @@ impl TokenStream { stream .iter() .enumerate() - .map(|(i, (tree, is_joint))| (f(i, tree.clone()), *is_joint)) + .map(|(i, tree)| f(i, tree.clone())) .collect()) })) } @@ -359,28 +339,17 @@ impl TokenStream { Lrc::new( stream .iter() - .map(|(tree, is_joint)| (f(tree.clone()), *is_joint)) + .map(|tree| f(tree.clone())) .collect()) })) } - fn first_tree_and_joint(&self) -> Option { - self.0.as_ref().map(|stream| { - stream.first().unwrap().clone() - }) + fn first_tree(&self) -> Option { + self.0.as_ref().map(|stream| stream.first().unwrap().clone()) } - fn last_tree_if_joint(&self) -> Option { - match self.0 { - None => None, - Some(ref stream) => { - if let (tree, Joint) = stream.last().unwrap() { - Some(tree.clone()) - } else { - None - } - } - } + fn last_tree(&self) -> Option { + self.0.as_ref().map(|stream| stream.last().unwrap().clone()) } } @@ -395,14 +364,14 @@ impl TokenStreamBuilder { pub fn push>(&mut self, stream: T) { let stream = stream.into(); - let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint); - if let Some(TokenTree::Token(last_token)) = last_tree_if_joint { - if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() { + let last_tree = self.0.last().and_then(TokenStream::last_tree); + if let Some(TokenTree::Token(last_token)) = last_tree { + if let Some(TokenTree::Token(token)) = stream.first_tree() { if let Some(glued_tok) = last_token.glue(&token) { let last_stream = self.0.pop().unwrap(); self.push_all_but_last_tree(&last_stream); let glued_tt = TokenTree::Token(glued_tok); - let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]); + let glued_tokenstream = TokenStream::new(vec![glued_tt]); self.0.push(glued_tokenstream); self.push_all_but_first_tree(&stream); return @@ -447,16 +416,6 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option { - self.next_with_joint().map(|(tree, _)| tree) - } -} - -impl Cursor { - fn new(stream: TokenStream) -> Self { - Cursor { stream, index: 0 } - } - - pub fn next_with_joint(&mut self) -> Option { match self.stream.0 { None => None, Some(ref stream) => { @@ -469,6 +428,13 @@ impl Cursor { } } } +} + +impl Cursor { + fn new(stream: TokenStream) -> Self { + Cursor { stream, index: 0 } + } + pub fn append(&mut self, new_stream: TokenStream) { if new_stream.is_empty() { @@ -483,7 +449,7 @@ impl Cursor { pub fn look_ahead(&self, n: usize) -> Option { match self.stream.0 { None => None, - Some(ref stream) => stream[self.index ..].get(n).map(|(tree, _)| tree.clone()), + Some(ref stream) => stream[self.index ..].get(n).cloned(), } } } diff --git a/src/libsyntax/tokenstream/tests.rs b/src/libsyntax/tokenstream/tests.rs index 5017e5f5424c1..e7f8e46038052 100644 --- a/src/libsyntax/tokenstream/tests.rs +++ b/src/libsyntax/tokenstream/tests.rs @@ -3,6 +3,7 @@ use super::*; use crate::ast::Name; use crate::with_default_globals; use crate::tests::string_to_stream; +use crate::parse::token::Joint; use syntax_pos::{Span, BytePos}; fn string_to_ts(string: &str) -> TokenStream { @@ -98,9 +99,9 @@ fn test_is_empty() { fn test_dotdotdot() { with_default_globals(|| { let mut builder = TokenStreamBuilder::new(); - builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint()); - builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint()); - builder.push(TokenTree::token(token::Dot, sp(2, 3))); + builder.push(TokenTree::Token(Token::new(token::Dot, sp(0, 1)).with_joint(Joint))); + builder.push(TokenTree::Token(Token::new(token::Dot, sp(1, 2)).with_joint(Joint))); + builder.push(TokenTree::Token(Token::new(token::Dot, sp(2, 3)))); let stream = builder.build(); assert!(stream.eq_unspanned(&string_to_ts("..."))); assert_eq!(stream.trees().count(), 1); diff --git a/src/libsyntax_ext/plugin_macro_defs.rs b/src/libsyntax_ext/plugin_macro_defs.rs index ccdc5bd81a04b..5253d74329f69 100644 --- a/src/libsyntax_ext/plugin_macro_defs.rs +++ b/src/libsyntax_ext/plugin_macro_defs.rs @@ -19,10 +19,10 @@ fn plugin_macro_def(name: Name, span: Span) -> P { let rustc_builtin_macro = attr::mk_attr_outer( attr::mk_word_item(Ident::new(sym::rustc_builtin_macro, span))); - let parens: TreeAndJoint = TokenTree::Delimited( + let parens = TokenTree::Delimited( DelimSpan::from_single(span), token::Paren, TokenStream::empty() - ).into(); - let trees = vec![parens.clone(), TokenTree::token(token::FatArrow, span).into(), parens]; + ); + let trees = vec![parens.clone(), TokenTree::token(token::FatArrow, span), parens]; P(Item { ident: Ident::new(name, span), diff --git a/src/test/ui/ast-json/ast-json-output.stdout b/src/test/ui/ast-json/ast-json-output.stdout index d23cbe0240ee1..a5a8d9001ea64 100644 --- a/src/test/ui/ast-json/ast-json-output.stdout +++ b/src/test/ui/ast-json/ast-json-output.stdout @@ -1 +1 @@ -{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"node":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]}]}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}} +{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"node":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0},"joint":"NonJoint"}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0},"joint":"NonJoint"}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0},"joint":"NonJoint"}]},{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0},"joint":"NonJoint"}]}]}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}