diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 851ab46345fd1..618a887489a8b 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -310,9 +310,10 @@ impl MetaItem { Some(&TokenTree::Token( Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span }, _, + _, )) => 'arm: { let mut segments = if let &token::Ident(name, _) = kind { - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) = + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) = tokens.peek() { tokens.next(); @@ -324,14 +325,17 @@ impl MetaItem { thin_vec![PathSegment::path_root(span)] }; loop { - if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) = - tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref() + if let Some(&TokenTree::Token( + Token { kind: token::Ident(name, _), span }, + _, + _, + )) = tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref() { segments.push(PathSegment::from_ident(Ident::new(name, span))); } else { return None; } - if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) = + if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) = tokens.peek() { tokens.next(); @@ -342,11 +346,13 @@ impl MetaItem { let span = span.with_hi(segments.last().unwrap().ident.span.hi()); Path { span, segments, tokens: None } } - Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 { - token::Nonterminal::NtMeta(item) => return item.meta(item.path.span), - token::Nonterminal::NtPath(path) => (**path).clone(), - _ => return None, - }, + Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _, _)) => { + match &nt.0 { + token::Nonterminal::NtMeta(item) => return item.meta(item.path.span), + token::Nonterminal::NtPath(path) => (**path).clone(), + _ => return None, + } + } _ => return None, }; let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi()); @@ -376,7 +382,7 @@ impl MetaItemKind { let item = NestedMetaItem::from_tokens(&mut tokens)?; result.push(item); match tokens.next() { - None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {} + None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => {} _ => return None, } } @@ -387,10 +393,10 @@ impl MetaItemKind { tokens: &mut impl Iterator, ) -> Option { match tokens.next() { - Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { + Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => { MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees()) } - Some(TokenTree::Token(token, _)) => { + Some(TokenTree::Token(token, _, _)) => { MetaItemLit::from_token(token).map(MetaItemKind::NameValue) } _ => None, @@ -401,13 +407,13 @@ impl MetaItemKind { tokens: &mut iter::Peekable>, ) -> Option { match tokens.peek() { - Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => { + Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens, _)) => { let inner_tokens = inner_tokens.clone(); tokens.next(); MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List) } Some(TokenTree::Delimited(..)) => None, - Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => { + Some(TokenTree::Token(Token { kind: token::Eq, .. }, _, _)) => { tokens.next(); MetaItemKind::name_value_from_tokens(tokens) } @@ -520,11 +526,13 @@ impl NestedMetaItem { I: Iterator, { match tokens.peek() { - Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => { + Some(TokenTree::Token(token, _, _)) + if let Some(lit) = MetaItemLit::from_token(token) => + { tokens.next(); return Some(NestedMetaItem::Lit(lit)); } - Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { + Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => { tokens.next(); return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable()); } @@ -585,6 +593,7 @@ pub fn mk_attr_nested_word( let inner_tokens = TokenStream::new(vec![TokenTree::Token( Token::from_ast_ident(Ident::new(inner, span)), Spacing::Alone, + span, )]); let outer_ident = Ident::new(outer, span); let path = Path::from_ident(outer_ident); diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index c6a31fbdbc31c..999fc9cecd25a 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -706,10 +706,10 @@ pub fn visit_attr_tt(tt: &mut AttrTokenTree, vis: &mut T) { // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. pub fn visit_tt(tt: &mut TokenTree, vis: &mut T) { match tt { - TokenTree::Token(token, _) => { + TokenTree::Token(token, _, _) => { visit_token(token, vis); } - TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { + TokenTree::Delimited(DelimSpan { open, close }, _delim, tts, _) => { vis.visit_span(open); vis.visit_span(close); visit_tts(tts, vis); diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 48854bbae24cd..c1d3e2d8f687a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -44,9 +44,9 @@ use std::{cmp, fmt, iter, mem}; pub enum TokenTree { /// A single token. Should never be `OpenDelim` or `CloseDelim`, because /// delimiters are implicitly represented by `Delimited`. - Token(Token, Spacing), + Token(Token, Spacing, Span), /// A delimited sequence of token trees. - Delimited(DelimSpan, Delimiter, TokenStream), + Delimited(DelimSpan, Delimiter, TokenStream, Span), } // Ensure all fields of `TokenTree` are `DynSend` and `DynSync`. @@ -65,8 +65,10 @@ impl TokenTree { /// Checks if this `TokenTree` is equal to the other, regardless of span information. pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { - (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind, - (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { + (TokenTree::Token(token, _, _), TokenTree::Token(token2, _, _)) => { + token.kind == token2.kind + } + (TokenTree::Delimited(_, delim, tts, _), TokenTree::Delimited(_, delim2, tts2, _)) => { delim == delim2 && tts.eq_unspanned(tts2) } _ => false, @@ -76,7 +78,7 @@ impl TokenTree { /// Retrieves the `TokenTree`'s span. pub fn span(&self) -> Span { match self { - TokenTree::Token(token, _) => token.span, + TokenTree::Token(token, _, _) => token.span, TokenTree::Delimited(sp, ..) => sp.entire(), } } @@ -84,25 +86,25 @@ impl TokenTree { /// Modify the `TokenTree`'s span in-place. pub fn set_span(&mut self, span: Span) { match self { - TokenTree::Token(token, _) => token.span = span, + TokenTree::Token(token, _, _) => token.span = span, TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), } } /// Create a `TokenTree::Token` with alone spacing. pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span), Spacing::Alone) + TokenTree::Token(Token::new(kind, span), Spacing::Alone, span) } /// Create a `TokenTree::Token` with joint spacing. pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree { - TokenTree::Token(Token::new(kind, span), Spacing::Joint) + TokenTree::Token(Token::new(kind, span), Spacing::Joint, span) } pub fn uninterpolate(&self) -> Cow<'_, TokenTree> { match self { - TokenTree::Token(token, spacing) => match token.uninterpolate() { - Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)), + TokenTree::Token(token, spacing, span) => match token.uninterpolate() { + Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing, *span)), Cow::Borrowed(_) => Cow::Borrowed(self), }, _ => Cow::Borrowed(self), @@ -206,12 +208,15 @@ impl AttrTokenStream { .iter() .flat_map(|tree| match &tree { AttrTokenTree::Token(inner, spacing) => { - smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() - } - AttrTokenTree::Delimited(span, delim, stream) => { - smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] - .into_iter() + smallvec![TokenTree::Token(inner.clone(), *spacing, inner.span)].into_iter() } + AttrTokenTree::Delimited(span, delim, stream) => smallvec![TokenTree::Delimited( + *span, + *delim, + stream.to_tokenstream(), + span.entire() + ),] + .into_iter(), AttrTokenTree::Attributes(data) => { let idx = data .attrs @@ -230,7 +235,7 @@ impl AttrTokenStream { let mut found = false; // Check the last two trees (to account for a trailing semi) for tree in target_tokens.iter_mut().rev().take(2) { - if let TokenTree::Delimited(span, delim, delim_tokens) = tree { + if let TokenTree::Delimited(span, delim, delim_tokens, _) = tree { // Inner attributes are only supported on extern blocks, functions, // impls, and modules. All of these have their inner attributes // placed at the beginning of the rightmost outermost braced group: @@ -250,7 +255,7 @@ impl AttrTokenStream { stream.push_stream(inner_attr.tokens()); } stream.push_stream(delim_tokens.clone()); - *tree = TokenTree::Delimited(*span, *delim, stream); + *tree = TokenTree::Delimited(*span, *delim, stream, span.entire()); found = true; break; } @@ -330,10 +335,10 @@ impl TokenStream { while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts, &next) { - (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, + (_, TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => continue, ( - TokenTree::Token(token_left, Spacing::Alone), - TokenTree::Token(token_right, _), + TokenTree::Token(token_left, Spacing::Alone, _), + TokenTree::Token(token_right, _, _), ) if ((token_left.is_ident() && !token_left.is_reserved_ident()) || token_left.is_lit()) && ((token_right.is_ident() && !token_right.is_reserved_ident()) @@ -433,7 +438,7 @@ impl TokenStream { /// Create a token stream containing a single `Delimited`. pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream { - TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)]) + TokenStream::new(vec![TokenTree::Delimited(span, delim, tts, span.entire())]) } pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { @@ -475,25 +480,32 @@ impl TokenStream { } } - fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { + fn flatten_token(token: &Token, spacing: Spacing, span: Span) -> TokenTree { match &token.kind { token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => { - TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) + TokenTree::Token( + Token::new(token::Ident(ident.name, is_raw), ident.span), + spacing, + token.span, + ) } token::Interpolated(nt) => TokenTree::Delimited( DelimSpan::from_single(token.span), Delimiter::Invisible, TokenStream::from_nonterminal_ast(&nt.0).flattened(), + token.span, ), - _ => TokenTree::Token(token.clone(), spacing), + _ => TokenTree::Token(token.clone(), spacing, span), } } fn flatten_token_tree(tree: &TokenTree) -> TokenTree { match tree { - TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing), - TokenTree::Delimited(span, delim, tts) => { - TokenTree::Delimited(*span, *delim, tts.flattened()) + TokenTree::Token(token, spacing, span) => { + TokenStream::flatten_token(token, *spacing, *span) + } + TokenTree::Delimited(span, delim, tts, span2) => { + TokenTree::Delimited(*span, *delim, tts.flattened(), *span2) } } } @@ -502,8 +514,8 @@ impl TokenStream { pub fn flattened(&self) -> TokenStream { fn can_skip(stream: &TokenStream) -> bool { stream.trees().all(|tree| match tree { - TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), - TokenTree::Delimited(_, _, inner) => can_skip(inner), + TokenTree::Token(token, _, _) => !matches!(token.kind, token::Interpolated(_)), + TokenTree::Delimited(_, _, inner, _) => can_skip(inner), }) } @@ -517,13 +529,13 @@ impl TokenStream { // If `vec` is not empty, try to glue `tt` onto its last token. The return // value indicates if gluing took place. fn try_glue_to_last(vec: &mut Vec, tt: &TokenTree) -> bool { - if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last() - && let TokenTree::Token(tok, spacing) = tt + if let Some(TokenTree::Token(last_tok, Spacing::Joint, _)) = vec.last() + && let TokenTree::Token(tok, spacing, span) = tt && let Some(glued_tok) = last_tok.glue(tok) { // ...then overwrite the last token tree in `vec` with the // glued token, and skip the first token tree from `stream`. - *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing); + *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing, *span); true } else { false @@ -582,6 +594,7 @@ impl TokenStream { &TokenTree::Token( Token { kind: token::DocComment(_, attr_style, data), span }, _spacing, + _, ) => { let desugared = desugared_tts(attr_style, data, span); let desugared_len = desugared.len(); @@ -592,9 +605,14 @@ impl TokenStream { &TokenTree::Token(..) => i += 1, - &TokenTree::Delimited(sp, delim, ref delim_stream) => { + &TokenTree::Delimited(sp, delim, ref delim_stream, _) => { if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) { - let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream); + let new_tt = TokenTree::Delimited( + sp, + delim, + desugared_delim_stream, + sp.entire(), + ); Lrc::make_mut(&mut stream.0)[i] = new_tt; modified = true; } @@ -637,6 +655,7 @@ impl TokenStream { ] .into_iter() .collect::(), + delim_span.entire(), ); if attr_style == AttrStyle::Inner { @@ -748,6 +767,6 @@ mod size_asserts { static_assert_size!(AttrTokenTree, 32); static_assert_size!(LazyAttrTokenStream, 8); static_assert_size!(TokenStream, 8); - static_assert_size!(TokenTree, 32); + static_assert_size!(TokenTree, 40); // tidy-alphabetical-end } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 1ad28ffbf2bb6..64d79ff030ace 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -164,11 +164,11 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool { // this match. match (tt1, tt2) { // No space after line doc comments. - (Tok(Token { kind: DocComment(CommentKind::Line, ..), .. }, _), _) => false, + (Tok(Token { kind: DocComment(CommentKind::Line, ..), .. }, _, _), _) => false, // `.` + ANYTHING: `x.y`, `tup.0` // `$` + ANYTHING: `$e` - (Tok(Token { kind: Dot | Dollar, .. }, _), _) => false, + (Tok(Token { kind: Dot | Dollar, .. }, _, _), _) => false, // ANYTHING + `,`: `foo,` // ANYTHING + `.`: `x.y`, `tup.0` @@ -177,16 +177,16 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool { // FIXME: Incorrect cases: // - Logical not: `x =! y`, `if! x { f(); }` // - Never type: `Fn() ->!` - (_, Tok(Token { kind: Comma | Dot | Not, .. }, _)) => false, + (_, Tok(Token { kind: Comma | Dot | Not, .. }, _, _)) => false, // IDENT + `(`: `f(3)` // // FIXME: Incorrect cases: // - Let: `let(a, b) = (1, 2)` - (Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false, + (Tok(Token { kind: Ident(..), .. }, _, _), Del(_, Parenthesis, _, _)) => false, // `#` + `[`: `#[attr]` - (Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false, + (Tok(Token { kind: Pound, .. }, _, _), Del(_, Bracket, _, _)) => false, _ => true, } @@ -511,14 +511,14 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere /// expression arguments as expressions). It can be done! I think. fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) { match tt { - TokenTree::Token(token, _) => { + TokenTree::Token(token, _, _) => { let token_str = self.token_to_string_ext(token, convert_dollar_crate); self.word(token_str); if let token::DocComment(..) = token.kind { self.hardbreak() } } - TokenTree::Delimited(dspan, delim, tts) => { + TokenTree::Delimited(dspan, delim, tts, _) => { self.print_mac_common( None, false, diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs index ee56d45c9c826..4dd08d360d37a 100644 --- a/compiler/rustc_builtin_macros/src/concat_idents.rs +++ b/compiler/rustc_builtin_macros/src/concat_idents.rs @@ -22,14 +22,14 @@ pub fn expand_concat_idents<'cx>( for (i, e) in tts.trees().enumerate() { if i & 1 == 1 { match e { - TokenTree::Token(Token { kind: token::Comma, .. }, _) => {} + TokenTree::Token(Token { kind: token::Comma, .. }, _, _) => {} _ => { cx.emit_err(errors::ConcatIdentsMissingComma { span: sp }); return DummyResult::any(sp); } } } else { - if let TokenTree::Token(token, _) = e { + if let TokenTree::Token(token, _, _) = e { if let Some((ident, _)) = token.ident() { res_str.push_str(ident.name.as_str()); continue; diff --git a/compiler/rustc_builtin_macros/src/trace_macros.rs b/compiler/rustc_builtin_macros/src/trace_macros.rs index af1a392acc5e5..3e2047f39d6e2 100644 --- a/compiler/rustc_builtin_macros/src/trace_macros.rs +++ b/compiler/rustc_builtin_macros/src/trace_macros.rs @@ -12,8 +12,8 @@ pub fn expand_trace_macros( let mut cursor = tt.trees(); let mut err = false; let value = match &cursor.next() { - Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true, - Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false, + Some(TokenTree::Token(token, _, _)) if token.is_keyword(kw::True) => true, + Some(TokenTree::Token(token, _, _)) if token.is_keyword(kw::False) => false, _ => { err = true; false diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 5ccef343b1723..9e1e1fb6862a7 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -365,7 +365,7 @@ impl<'a> StripUnconfigured<'a> { // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // for `attr` when we expand it to `#[attr]` let mut orig_trees = orig_tokens.trees(); - let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = + let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _, _) = orig_trees.next().unwrap().clone() else { panic!("Bad tokens for attribute {attr:?}"); @@ -375,7 +375,7 @@ impl<'a> StripUnconfigured<'a> { let mut trees = vec![AttrTokenTree::Token(pound_token, Spacing::Alone)]; if attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` - let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = + let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _, _) = orig_trees.next().unwrap().clone() else { panic!("Bad tokens for attribute {attr:?}"); diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs index 7cb279a981203..b529402e09096 100644 --- a/compiler/rustc_expand/src/mbe/metavar_expr.rs +++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs @@ -35,7 +35,7 @@ impl MetaVarExpr { ) -> PResult<'sess, MetaVarExpr> { let mut tts = input.trees(); let ident = parse_ident(&mut tts, sess, outer_span)?; - let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args)) = tts.next() else { + let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args, _)) = tts.next() else { let msg = "meta-variable expression parameter must be wrapped in parentheses"; return Err(sess.span_diagnostic.struct_span_err(ident.span, msg)); }; @@ -114,7 +114,8 @@ fn parse_depth<'sess>( span: Span, ) -> PResult<'sess, usize> { let Some(tt) = iter.next() else { return Ok(0) }; - let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else { + let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _, _) = tt + else { return Err(sess .span_diagnostic .struct_span_err(span, "meta-variable expression depth must be a literal")); @@ -137,7 +138,7 @@ fn parse_ident<'sess>( span: Span, ) -> PResult<'sess, Ident> { if let Some(tt) = iter.next() - && let TokenTree::Token(token, _) = tt + && let TokenTree::Token(token, _, _) = tt { if let Some((elem, false)) = token.ident() { return Ok(elem); @@ -160,7 +161,9 @@ fn parse_ident<'sess>( /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// iterator is not modified and the result is `false`. fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool { - if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) { + if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _, _)) = + iter.look_ahead(0) + { let _ = iter.next(); return true; } diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 6a99412fc5bcf..b58d835a47390 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -56,9 +56,14 @@ pub(super) fn parse( match tree { TokenTree::MetaVar(start_sp, ident) if parsing_patterns => { let span = match trees.next() { - Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => { + Some(&tokenstream::TokenTree::Token( + Token { kind: token::Colon, span }, + _, + _, + )) => { match trees.next() { - Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() { + Some(tokenstream::TokenTree::Token(token, _, _)) => match token.ident() + { Some((frag, _)) => { let span = token.span.with_lo(start_sp.lo()); @@ -146,12 +151,12 @@ fn parse_tree<'a>( // Depending on what `tree` is, we could be parsing different parts of a macro match tree { // `tree` is a `$` token. Look at the next token in `trees` - &tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => { + &tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _, _) => { // FIXME: Handle `Invisible`-delimited groups in a more systematic way // during parsing. let mut next = outer_trees.next(); let mut trees: Box>; - if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next { + if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts, _)) = next { trees = Box::new(tts.trees()); next = trees.next(); } else { @@ -160,7 +165,7 @@ fn parse_tree<'a>( match next { // `tree` is followed by a delimited set of token trees. - Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => { + Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts, _)) => { if parsing_patterns { if delim != Delimiter::Parenthesis { span_dollar_dollar_or_metavar_in_the_lhs_err( @@ -217,7 +222,7 @@ fn parse_tree<'a>( // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` // special metavariable that names the crate of the invocation. - Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => { + Some(tokenstream::TokenTree::Token(token, _, _)) if token.is_ident() => { let (ident, is_raw) = token.ident().unwrap(); let span = ident.span.with_lo(span.lo()); if ident.name == kw::Crate && !is_raw { @@ -228,7 +233,7 @@ fn parse_tree<'a>( } // `tree` is followed by another `$`. This is an escaped `$`. - Some(&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => { + Some(&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _, _)) => { if parsing_patterns { span_dollar_dollar_or_metavar_in_the_lhs_err( sess, @@ -241,7 +246,7 @@ fn parse_tree<'a>( } // `tree` is followed by some other token. This is an error. - Some(tokenstream::TokenTree::Token(token, _)) => { + Some(tokenstream::TokenTree::Token(token, _, _)) => { let msg = format!("expected identifier, found `{}`", pprust::token_to_string(token),); sess.span_diagnostic.span_err(token.span, msg); @@ -254,11 +259,11 @@ fn parse_tree<'a>( } // `tree` is an arbitrary token. Keep it. - tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()), + tokenstream::TokenTree::Token(token, _, _) => TokenTree::Token(token.clone()), // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // descend into the delimited set and further parse it. - &tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited( + &tokenstream::TokenTree::Delimited(span, delim, ref tts, _) => TokenTree::Delimited( span, Delimited { delim, @@ -289,7 +294,7 @@ fn parse_kleene_op<'a>( span: Span, ) -> Result, Span> { match input.next() { - Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(token) { + Some(tokenstream::TokenTree::Token(token, _, _)) => match kleene_op(token) { Some(op) => Ok(Ok((op, token.span))), None => Ok(Err(token.clone())), }, diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index bc03fc0d1b169..f5932ff12ae92 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -126,7 +126,7 @@ pub(super) fn transcribe<'a>( if repeat_idx < repeat_len { *idx = 0; if let Some(sep) = sep { - result.push(TokenTree::Token(sep.clone(), Spacing::Alone)); + result.push(TokenTree::Token(sep.clone(), Spacing::Alone, sep.span)); } continue; } @@ -151,7 +151,8 @@ pub(super) fn transcribe<'a>( } // Step back into the parent Delimited. - let tree = TokenTree::Delimited(span, delim, TokenStream::new(result)); + let tree = + TokenTree::Delimited(span, delim, TokenStream::new(result), span.entire()); result = result_stack.pop().unwrap(); result.push(tree); } @@ -244,6 +245,7 @@ pub(super) fn transcribe<'a>( result.push(TokenTree::Token( Token::from_ast_ident(original_ident), Spacing::Alone, + original_ident.span, )); } } @@ -274,7 +276,8 @@ pub(super) fn transcribe<'a>( mbe::TokenTree::Token(token) => { let mut token = token.clone(); mut_visit::visit_token(&mut token, &mut marker); - let tt = TokenTree::Token(token, Spacing::Alone); + let span = token.span; + let tt = TokenTree::Token(token, Spacing::Alone, span); result.push(tt); } diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs index bdc20882a9d52..d4615835eef7a 100644 --- a/compiler/rustc_expand/src/parse/tests.rs +++ b/compiler/rustc_expand/src/parse/tests.rs @@ -77,20 +77,24 @@ fn string_to_tts_macro() { TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _), TokenTree::Token(Token { kind: token::Not, .. }, _), TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _), - TokenTree::Delimited(_, macro_delim, macro_tts), + TokenTree::Delimited(_, macro_delim, macro_tts, _), ] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => { let tts = ¯o_tts.trees().collect::>(); match &tts[..] { [ - TokenTree::Delimited(_, first_delim, first_tts), - TokenTree::Token(Token { kind: token::FatArrow, .. }, _), - TokenTree::Delimited(_, second_delim, second_tts), + TokenTree::Delimited(_, first_delim, first_tts, _), + TokenTree::Token(Token { kind: token::FatArrow, .. }, _, _), + TokenTree::Delimited(_, second_delim, second_tts, _), ] if macro_delim == &Delimiter::Parenthesis => { let tts = &first_tts.trees().collect::>(); match &tts[..] { [ - TokenTree::Token(Token { kind: token::Dollar, .. }, _), - TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _), + TokenTree::Token(Token { kind: token::Dollar, .. }, _, _), + TokenTree::Token( + Token { kind: token::Ident(name, false), .. }, + _, + _, + ), ] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => { } _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), @@ -130,6 +134,7 @@ fn string_to_tts_1() { TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)), ]) .into(), + _, ), TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), @@ -139,6 +144,7 @@ fn string_to_tts_1() { TokenTree::token_alone(token::Semi, sp(18, 19)), ]) .into(), + _, ), ]); diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index b057a645f8195..272b3d723104d 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -98,7 +98,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec { + tokenstream::TokenTree::Delimited(span, delim, tts, _) => { let delimiter = pm::Delimiter::from_internal(delim); trees.push(TokenTree::Group(Group { delimiter, @@ -111,7 +111,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec (token, spacing == Joint), + tokenstream::TokenTree::Token(token, spacing, _) => (token, spacing == Joint), }; // Split the operator into one or more `Punct`s, one per character. @@ -303,10 +303,12 @@ impl ToInternal> }] } TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { + let dspan = tokenstream::DelimSpan { open, close }; smallvec![tokenstream::TokenTree::Delimited( - tokenstream::DelimSpan { open, close }, + dspan, delimiter.to_internal(), stream.unwrap_or_default(), + dspan.entire() )] } TokenTree::Ident(self::Ident { sym, is_raw, span }) => { diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index 8932200c5b7b1..9d910e3567e14 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -1829,12 +1829,12 @@ impl KeywordIdents { for tt in tokens.trees() { match tt { // Only report non-raw idents. - TokenTree::Token(token, _) => { + TokenTree::Token(token, _, _) => { if let Some((ident, false)) = token.ident() { self.check_ident_token(cx, UnderMacro(true), ident); } } - TokenTree::Delimited(_, _, tts) => self.check_tokens(cx, tts), + TokenTree::Delimited(_, _, tts, _) => self.check_tokens(cx, tts), } } } diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index e646f5dfd8514..1cf80d8bdfe66 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -73,7 +73,8 @@ impl<'a> TokenTreesReader<'a> { } }; let this_tok = std::mem::replace(&mut self.token, next_tok); - buf.push(TokenTree::Token(this_tok, this_spacing)); + let span = this_tok.span; + buf.push(TokenTree::Token(this_tok, this_spacing, span)); } } } @@ -202,7 +203,7 @@ impl<'a> TokenTreesReader<'a> { _ => unreachable!(), } - Ok(TokenTree::Delimited(delim_span, open_delim, tts)) + Ok(TokenTree::Delimited(delim_span, open_delim, tts, delim_span.entire())) } fn unclosed_delim_err(&mut self, tts: TokenStream, mut errs: Vec>) -> Vec> { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 2816386cbad9f..a542281099b6e 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -257,14 +257,14 @@ impl TokenCursor { // below can be removed. if let Some(tree) = self.tree_cursor.next_ref() { match tree { - &TokenTree::Token(ref token, spacing) => { + &TokenTree::Token(ref token, spacing, _) => { debug_assert!(!matches!( token.kind, token::OpenDelim(_) | token::CloseDelim(_) )); return (token.clone(), spacing); } - &TokenTree::Delimited(sp, delim, ref tts) => { + &TokenTree::Delimited(sp, delim, ref tts, _) => { let trees = tts.clone().into_trees(); self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp)); if delim != Delimiter::Invisible { @@ -1077,7 +1077,7 @@ impl<'a> Parser<'a> { let tree_cursor = &self.token_cursor.tree_cursor; let all_normal = (0..dist).all(|i| { let token = tree_cursor.look_ahead(i); - !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _))) + !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _, _))) }); if all_normal { // There were no skipped delimiters. Do lookahead by plain indexing. @@ -1085,8 +1085,8 @@ impl<'a> Parser<'a> { Some(tree) => { // Indexing stayed within the current token stream. match tree { - TokenTree::Token(token, _) => looker(token), - TokenTree::Delimited(dspan, delim, _) => { + TokenTree::Token(token, _, _) => looker(token), + TokenTree::Delimited(dspan, delim, _, _) => { looker(&Token::new(token::OpenDelim(*delim), dspan.open)) } } @@ -1257,7 +1257,7 @@ impl<'a> Parser<'a> { || self.check(&token::OpenDelim(Delimiter::Brace)); delimited.then(|| { - let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else { + let TokenTree::Delimited(dspan, delim, tokens, _) = self.parse_token_tree() else { unreachable!() }; DelimArgs { dspan, delim, tokens } @@ -1301,12 +1301,12 @@ impl<'a> Parser<'a> { // Consume close delimiter self.bump(); - TokenTree::Delimited(span, delim, stream) + TokenTree::Delimited(span, delim, stream, span.entire()) } token::CloseDelim(_) | token::Eof => unreachable!(), _ => { self.bump(); - TokenTree::Token(self.prev_token.clone(), Spacing::Alone) + TokenTree::Token(self.prev_token.clone(), Spacing::Alone, self.prev_token.span) } } } diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index fe1f43835ef3a..41e902b80b029 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -2630,7 +2630,7 @@ fn filter_tokens_from_list( let mut skip_next_comma = false; for token in args_tokens.trees() { match token { - TokenTree::Token(Token { kind: TokenKind::Comma, .. }, _) if skip_next_comma => { + TokenTree::Token(Token { kind: TokenKind::Comma, .. }, _, _) if skip_next_comma => { skip_next_comma = false; } token if should_retain(token) => { @@ -2704,6 +2704,7 @@ fn add_without_unwanted_attributes<'hir>( .. }, _, + _ ), ) }); diff --git a/src/librustdoc/clean/render_macro_matchers.rs b/src/librustdoc/clean/render_macro_matchers.rs index 66d10f2368b22..28647806c4af5 100644 --- a/src/librustdoc/clean/render_macro_matchers.rs +++ b/src/librustdoc/clean/render_macro_matchers.rs @@ -40,7 +40,7 @@ pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Stri printer.zerobreak(); printer.ibox(0); match matcher { - TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), + TokenTree::Delimited(_span, _delim, tts, _) => print_tts(&mut printer, tts), // Matcher which is not a Delimited is unexpected and should've failed // to compile, but we render whatever it is wrapped in parens. TokenTree::Token(..) => print_tt(&mut printer, matcher), @@ -90,14 +90,14 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option, tt: &TokenTree) { match tt { - TokenTree::Token(token, _) => { + TokenTree::Token(token, _, _) => { let token_str = printer.token_to_string(token); printer.word(token_str); if let token::DocComment(..) = token.kind { printer.hardbreak() } } - TokenTree::Delimited(_span, delim, tts) => { + TokenTree::Delimited(_span, delim, tts, _) => { let open_delim = printer.token_kind_to_string(&token::OpenDelim(*delim)); printer.word(open_delim); if !tts.is_empty() { @@ -135,7 +135,7 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { let mut state = Start; for tt in tts.trees() { let (needs_space, next_state) = match &tt { - TokenTree::Token(tt, _) => match (state, &tt.kind) { + TokenTree::Token(tt, _, _) => match (state, &tt.kind) { (Dollar, token::Ident(..)) => (false, DollarIdent), (DollarIdent, token::Colon) => (false, DollarIdentColon), (DollarIdentColon, token::Ident(..)) => (false, Other), @@ -158,7 +158,7 @@ fn print_tts(printer: &mut Printer<'_>, tts: &TokenStream) { (_, token::Pound) => (true, Pound), (_, _) => (true, Other), }, - TokenTree::Delimited(_, delim, _) => match (state, delim) { + TokenTree::Delimited(_, delim, _, _) => match (state, delim) { (Dollar, Delimiter::Parenthesis) => (false, DollarParen), (Pound | PoundBang, Delimiter::Bracket) => (false, Other), (Ident, Delimiter::Parenthesis | Delimiter::Bracket) => (false, Other), diff --git a/src/tools/clippy/clippy_lints/src/attrs.rs b/src/tools/clippy/clippy_lints/src/attrs.rs index da38422874b44..9637fe62916fb 100644 --- a/src/tools/clippy/clippy_lints/src/attrs.rs +++ b/src/tools/clippy/clippy_lints/src/attrs.rs @@ -609,12 +609,14 @@ fn check_should_panic_reason(cx: &LateContext<'_>, attr: &Attribute) { .. }, _, + _ )) = tt_iter.next() && let Some(TokenTree::Token( Token { kind: TokenKind::Eq, .. }, _, + _ )) = tt_iter.next() && let Some(TokenTree::Token( Token { @@ -622,6 +624,7 @@ fn check_should_panic_reason(cx: &LateContext<'_>, attr: &Attribute) { .. }, _, + _ )) = tt_iter.next() { // `#[should_panic(expected = "..")]` found, good diff --git a/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs b/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs index d4828778be28e..7a29dd3721e75 100644 --- a/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs +++ b/src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs @@ -92,7 +92,7 @@ fn contains_unhygienic_crate_reference(tts: &TokenStream) -> Option { { return Some(span); } - if let TokenTree::Delimited(_, _, tts) = &curr { + if let TokenTree::Delimited(_, _, tts, _) = &curr { let span = contains_unhygienic_crate_reference(tts); if span.is_some() { return span; @@ -110,6 +110,7 @@ fn is_crate_keyword(tt: &TokenTree) -> Option { span, }, _, + _ ) = tt && symbol.as_str() == "crate" { @@ -120,7 +121,7 @@ fn is_crate_keyword(tt: &TokenTree) -> Option { } fn is_token(tt: &TokenTree, kind: &TokenKind) -> bool { - if let TokenTree::Token(Token { kind: other, .. }, _) = tt { + if let TokenTree::Token(Token { kind: other, .. }, _, _) = tt { kind == other } else { false diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index 76553466e4812..47a5827f57e80 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -707,8 +707,8 @@ struct MacroArgParser { fn last_tok(tt: &TokenTree) -> Token { match *tt { - TokenTree::Token(ref t, _) => t.clone(), - TokenTree::Delimited(delim_span, delim, _) => Token { + TokenTree::Token(ref t, _, _) => t.clone(), + TokenTree::Delimited(delim_span, delim, _, _) => Token { kind: TokenKind::CloseDelim(delim), span: delim_span.close, }, @@ -768,6 +768,7 @@ impl MacroArgParser { .. }, _, + _, )) => { self.result.push(ParsedMacroArg { kind: MacroArgKind::MetaVariable(name, self.buf.clone()), @@ -811,6 +812,7 @@ impl MacroArgParser { .. }, _, + _, ) | TokenTree::Token( Token { @@ -818,6 +820,7 @@ impl MacroArgParser { .. }, _, + _, ) | TokenTree::Token( Token { @@ -825,10 +828,11 @@ impl MacroArgParser { .. }, _, + _, ) => { break; } - TokenTree::Token(ref t, _) => { + TokenTree::Token(ref t, _, _) => { buffer.push_str(&pprust::token_to_string(t)); } _ => return None, @@ -902,6 +906,7 @@ impl MacroArgParser { span, }, _, + _, ) => { // We always want to add a separator before meta variables. if !self.buf.is_empty() { @@ -921,11 +926,12 @@ impl MacroArgParser { .. }, _, + _, ) if self.is_meta_var => { self.add_meta_variable(&mut iter)?; } - TokenTree::Token(ref t, _) => self.update_buffer(t), - &TokenTree::Delimited(_delimited_span, delimited, ref tts) => { + TokenTree::Token(ref t, _, _) => self.update_buffer(t), + &TokenTree::Delimited(_delimited_span, delimited, ref tts, _) => { if !self.buf.is_empty() { if next_space(&self.last_tok.kind) == SpaceState::Always { self.add_separator(); @@ -1167,7 +1173,7 @@ impl<'a> MacroParser<'a> { let tok = self.toks.next()?; let (lo, args_paren_kind) = match tok { TokenTree::Token(..) => return None, - &TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d), + &TokenTree::Delimited(delimited_span, d, _, _) => (delimited_span.open.lo(), d), }; let args = TokenStream::new(vec![tok.clone()]); match self.toks.next()? { @@ -1177,6 +1183,7 @@ impl<'a> MacroParser<'a> { .. }, _, + _, ) => {} _ => return None, } @@ -1202,6 +1209,7 @@ impl<'a> MacroParser<'a> { span, }, _, + _, )) = self.toks.look_ahead(0) { hi = span.hi();