Skip to content

[PERF-EXPERIMENT] Add second span to TokenTree #118671

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[PERF-EXPERIMENT] Add second span to TokenTree #118671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 25 additions & 16 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,10 @@ impl MetaItem {
Some(&TokenTree::Token(
Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span },
_,
_,
)) => 'arm: {
let mut segments = if let &token::Ident(name, _) = kind {
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) =
tokens.peek()
{
tokens.next();
Expand All @@ -324,14 +325,17 @@ impl MetaItem {
thin_vec![PathSegment::path_root(span)]
};
loop {
if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
if let Some(&TokenTree::Token(
Token { kind: token::Ident(name, _), span },
_,
_,
)) = tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
{
segments.push(PathSegment::from_ident(Ident::new(name, span)));
} else {
return None;
}
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _, _)) =
tokens.peek()
{
tokens.next();
Expand All @@ -342,11 +346,13 @@ impl MetaItem {
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments, tokens: None }
}
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 {
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(path) => (**path).clone(),
_ => return None,
},
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _, _)) => {
match &nt.0 {
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(path) => (**path).clone(),
_ => return None,
}
}
_ => return None,
};
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
Expand Down Expand Up @@ -376,7 +382,7 @@ impl MetaItemKind {
let item = NestedMetaItem::from_tokens(&mut tokens)?;
result.push(item);
match tokens.next() {
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => {}
_ => return None,
}
}
Expand All @@ -387,10 +393,10 @@ impl MetaItemKind {
tokens: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> {
match tokens.next() {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
}
Some(TokenTree::Token(token, _)) => {
Some(TokenTree::Token(token, _, _)) => {
MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
}
_ => None,
Expand All @@ -401,13 +407,13 @@ impl MetaItemKind {
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> {
match tokens.peek() {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens, _)) => {
let inner_tokens = inner_tokens.clone();
tokens.next();
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
}
Some(TokenTree::Delimited(..)) => None,
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _, _)) => {
tokens.next();
MetaItemKind::name_value_from_tokens(tokens)
}
Expand Down Expand Up @@ -520,11 +526,13 @@ impl NestedMetaItem {
I: Iterator<Item = &'a TokenTree>,
{
match tokens.peek() {
Some(TokenTree::Token(token, _)) if let Some(lit) = MetaItemLit::from_token(token) => {
Some(TokenTree::Token(token, _, _))
if let Some(lit) = MetaItemLit::from_token(token) =>
{
tokens.next();
return Some(NestedMetaItem::Lit(lit));
}
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens, _)) => {
tokens.next();
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
}
Expand Down Expand Up @@ -585,6 +593,7 @@ pub fn mk_attr_nested_word(
let inner_tokens = TokenStream::new(vec![TokenTree::Token(
Token::from_ast_ident(Ident::new(inner, span)),
Spacing::Alone,
span,
)]);
let outer_ident = Ident::new(outer, span);
let path = Path::from_ident(outer_ident);
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_ast/src/mut_visit.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,10 +706,10 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
match tt {
TokenTree::Token(token, _) => {
TokenTree::Token(token, _, _) => {
visit_token(token, vis);
}
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts, _) => {
vis.visit_span(open);
vis.visit_span(close);
visit_tts(tts, vis);
Expand Down
89 changes: 54 additions & 35 deletions compiler/rustc_ast/src/tokenstream.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ use std::{cmp, fmt, iter, mem};
pub enum TokenTree {
/// A single token. Should never be `OpenDelim` or `CloseDelim`, because
/// delimiters are implicitly represented by `Delimited`.
Token(Token, Spacing),
Token(Token, Spacing, Span),
/// A delimited sequence of token trees.
Delimited(DelimSpan, Delimiter, TokenStream),
Delimited(DelimSpan, Delimiter, TokenStream, Span),
}

// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
Expand All @@ -65,8 +65,10 @@ impl TokenTree {
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
match (self, other) {
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
(TokenTree::Token(token, _, _), TokenTree::Token(token2, _, _)) => {
token.kind == token2.kind
}
(TokenTree::Delimited(_, delim, tts, _), TokenTree::Delimited(_, delim2, tts2, _)) => {
delim == delim2 && tts.eq_unspanned(tts2)
}
_ => false,
Expand All @@ -76,33 +78,33 @@ impl TokenTree {
/// Retrieves the `TokenTree`'s span.
pub fn span(&self) -> Span {
match self {
TokenTree::Token(token, _) => token.span,
TokenTree::Token(token, _, _) => token.span,
TokenTree::Delimited(sp, ..) => sp.entire(),
}
}

/// Modify the `TokenTree`'s span in-place.
pub fn set_span(&mut self, span: Span) {
match self {
TokenTree::Token(token, _) => token.span = span,
TokenTree::Token(token, _, _) => token.span = span,
TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span),
}
}

/// Create a `TokenTree::Token` with alone spacing.
pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
TokenTree::Token(Token::new(kind, span), Spacing::Alone, span)
}

/// Create a `TokenTree::Token` with joint spacing.
pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
TokenTree::Token(Token::new(kind, span), Spacing::Joint, span)
}

pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
match self {
TokenTree::Token(token, spacing) => match token.uninterpolate() {
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
TokenTree::Token(token, spacing, span) => match token.uninterpolate() {
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing, *span)),
Cow::Borrowed(_) => Cow::Borrowed(self),
},
_ => Cow::Borrowed(self),
Expand Down Expand Up @@ -206,12 +208,15 @@ impl AttrTokenStream {
.iter()
.flat_map(|tree| match &tree {
AttrTokenTree::Token(inner, spacing) => {
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
}
AttrTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
.into_iter()
smallvec![TokenTree::Token(inner.clone(), *spacing, inner.span)].into_iter()
}
AttrTokenTree::Delimited(span, delim, stream) => smallvec![TokenTree::Delimited(
*span,
*delim,
stream.to_tokenstream(),
span.entire()
),]
.into_iter(),
AttrTokenTree::Attributes(data) => {
let idx = data
.attrs
Expand All @@ -230,7 +235,7 @@ impl AttrTokenStream {
let mut found = false;
// Check the last two trees (to account for a trailing semi)
for tree in target_tokens.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
if let TokenTree::Delimited(span, delim, delim_tokens, _) = tree {
// Inner attributes are only supported on extern blocks, functions,
// impls, and modules. All of these have their inner attributes
// placed at the beginning of the rightmost outermost braced group:
Expand All @@ -250,7 +255,7 @@ impl AttrTokenStream {
stream.push_stream(inner_attr.tokens());
}
stream.push_stream(delim_tokens.clone());
*tree = TokenTree::Delimited(*span, *delim, stream);
*tree = TokenTree::Delimited(*span, *delim, stream, span.entire());
found = true;
break;
}
Expand Down Expand Up @@ -330,10 +335,10 @@ impl TokenStream {
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _, _)) => continue,
(
TokenTree::Token(token_left, Spacing::Alone),
TokenTree::Token(token_right, _),
TokenTree::Token(token_left, Spacing::Alone, _),
TokenTree::Token(token_right, _, _),
) if ((token_left.is_ident() && !token_left.is_reserved_ident())
|| token_left.is_lit())
&& ((token_right.is_ident() && !token_right.is_reserved_ident())
Expand Down Expand Up @@ -433,7 +438,7 @@ impl TokenStream {

/// Create a token stream containing a single `Delimited`.
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts, span.entire())])
}

pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
Expand Down Expand Up @@ -475,25 +480,32 @@ impl TokenStream {
}
}

fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
fn flatten_token(token: &Token, spacing: Spacing, span: Span) -> TokenTree {
match &token.kind {
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => {
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
TokenTree::Token(
Token::new(token::Ident(ident.name, is_raw), ident.span),
spacing,
token.span,
)
}
token::Interpolated(nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
Delimiter::Invisible,
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
token.span,
),
_ => TokenTree::Token(token.clone(), spacing),
_ => TokenTree::Token(token.clone(), spacing, span),
}
}

fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
match tree {
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
TokenTree::Delimited(span, delim, tts) => {
TokenTree::Delimited(*span, *delim, tts.flattened())
TokenTree::Token(token, spacing, span) => {
TokenStream::flatten_token(token, *spacing, *span)
}
TokenTree::Delimited(span, delim, tts, span2) => {
TokenTree::Delimited(*span, *delim, tts.flattened(), *span2)
}
}
}
Expand All @@ -502,8 +514,8 @@ impl TokenStream {
pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
TokenTree::Delimited(_, _, inner) => can_skip(inner),
TokenTree::Token(token, _, _) => !matches!(token.kind, token::Interpolated(_)),
TokenTree::Delimited(_, _, inner, _) => can_skip(inner),
})
}

Expand All @@ -517,13 +529,13 @@ impl TokenStream {
// If `vec` is not empty, try to glue `tt` onto its last token. The return
// value indicates if gluing took place.
fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
&& let TokenTree::Token(tok, spacing) = tt
if let Some(TokenTree::Token(last_tok, Spacing::Joint, _)) = vec.last()
&& let TokenTree::Token(tok, spacing, span) = tt
&& let Some(glued_tok) = last_tok.glue(tok)
{
// ...then overwrite the last token tree in `vec` with the
// glued token, and skip the first token tree from `stream`.
*vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
*vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing, *span);
true
} else {
false
Expand Down Expand Up @@ -582,6 +594,7 @@ impl TokenStream {
&TokenTree::Token(
Token { kind: token::DocComment(_, attr_style, data), span },
_spacing,
_,
) => {
let desugared = desugared_tts(attr_style, data, span);
let desugared_len = desugared.len();
Expand All @@ -592,9 +605,14 @@ impl TokenStream {

&TokenTree::Token(..) => i += 1,

&TokenTree::Delimited(sp, delim, ref delim_stream) => {
&TokenTree::Delimited(sp, delim, ref delim_stream, _) => {
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
let new_tt = TokenTree::Delimited(
sp,
delim,
desugared_delim_stream,
sp.entire(),
);
Lrc::make_mut(&mut stream.0)[i] = new_tt;
modified = true;
}
Expand Down Expand Up @@ -637,6 +655,7 @@ impl TokenStream {
]
.into_iter()
.collect::<TokenStream>(),
delim_span.entire(),
);

if attr_style == AttrStyle::Inner {
Expand Down Expand Up @@ -748,6 +767,6 @@ mod size_asserts {
static_assert_size!(AttrTokenTree, 32);
static_assert_size!(LazyAttrTokenStream, 8);
static_assert_size!(TokenStream, 8);
static_assert_size!(TokenTree, 32);
static_assert_size!(TokenTree, 40);
// tidy-alphabetical-end
}
Loading