diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 75c656973f963..4cfac79374979 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -26,7 +26,7 @@ pub use UnsafeSource::*; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter}; -use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream}; +use crate::tokenstream::{AttrTokenStream, DelimSpan, TokenStream}; pub use rustc_ast_ir::{Movability, Mutability}; use rustc_data_structures::packed::Pu128; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; @@ -94,7 +94,7 @@ pub struct Path { /// The segments in the path: the things separated by `::`. /// Global paths begin with `kw::PathRoot`. pub segments: ThinVec, - pub tokens: Option, + pub tokens: Option, } impl PartialEq for Path { @@ -544,7 +544,7 @@ pub struct Block { /// Distinguishes between `unsafe { ... }` and `{ ... }`. pub rules: BlockCheckMode, pub span: Span, - pub tokens: Option, + pub tokens: Option, /// The following *isn't* a parse error, but will cause multiple errors in following stages. /// ```compile_fail /// let x = { @@ -563,7 +563,7 @@ pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } impl Pat { @@ -1002,7 +1002,7 @@ impl Stmt { /// a trailing semicolon. /// /// This only modifies the parsed AST struct, not the attached - /// `LazyAttrTokenStream`. The parser is responsible for calling + /// `AttrTokenStream`. The parser is responsible for calling /// `ToAttrTokenStream::add_trailing_semi` when there is actually /// a semicolon in the tokenstream. pub fn add_trailing_semicolon(mut self) -> Self { @@ -1050,7 +1050,7 @@ pub struct MacCallStmt { pub mac: P, pub style: MacStmtStyle, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)] @@ -1076,7 +1076,7 @@ pub struct Local { pub span: Span, pub colon_sp: Option, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -1175,7 +1175,7 @@ pub struct Expr { pub kind: ExprKind, pub span: Span, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } impl Expr { @@ -2102,7 +2102,7 @@ pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } impl Clone for Ty { @@ -2831,7 +2831,7 @@ pub enum AttrKind { pub struct NormalAttr { pub item: AttrItem, // Tokens for the full attribute, e.g. `#[foo]`, `#![bar]`. - pub tokens: Option, + pub tokens: Option, } impl NormalAttr { @@ -2854,7 +2854,7 @@ pub struct AttrItem { pub path: Path, pub args: AttrArgs, // Tokens for the meta item, e.g. just the `foo` within `#[foo]` or `#![foo]`. - pub tokens: Option, + pub tokens: Option, } /// `TraitRef`s appear in impls. @@ -2894,7 +2894,7 @@ impl PolyTraitRef { pub struct Visibility { pub kind: VisibilityKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2987,7 +2987,7 @@ pub struct Item { /// /// Note that the tokens here do not include the outer attributes, but will /// include inner attributes. - pub tokens: Option, + pub tokens: Option, } impl Item { diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index 7754ca0a0f503..3f146b6d8c2b5 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -4,7 +4,7 @@ use crate::ptr::P; use crate::token::Nonterminal; -use crate::tokenstream::LazyAttrTokenStream; +use crate::tokenstream::AttrTokenStream; use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant}; use crate::{AssocItem, Expr, ForeignItem, Item, NodeId}; use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility}; @@ -91,18 +91,18 @@ impl> HasNodeId for T { /// A trait for AST nodes having (or not having) collected tokens. pub trait HasTokens { - fn tokens(&self) -> Option<&LazyAttrTokenStream>; - fn tokens_mut(&mut self) -> Option<&mut Option>; + fn tokens(&self) -> Option<&AttrTokenStream>; + fn tokens_mut(&mut self) -> Option<&mut Option>; } macro_rules! impl_has_tokens { ($($T:ty),+ $(,)?) => { $( impl HasTokens for $T { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { self.tokens.as_ref() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { Some(&mut self.tokens) } } @@ -114,10 +114,10 @@ macro_rules! impl_has_tokens_none { ($($T:ty),+ $(,)?) => { $( impl HasTokens for $T { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { None } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { None } } @@ -129,25 +129,25 @@ impl_has_tokens!(AssocItem, AttrItem, Block, Expr, ForeignItem, Item, Pat, Path, impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant); impl> HasTokens for T { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { self.ast_deref().tokens() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.ast_deref_mut().tokens_mut() } } impl HasTokens for Option { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { self.as_ref().and_then(|inner| inner.tokens()) } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.as_mut().and_then(|inner| inner.tokens_mut()) } } impl HasTokens for StmtKind { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { match self { StmtKind::Let(local) => local.tokens.as_ref(), StmtKind::Item(item) => item.tokens(), @@ -156,7 +156,7 @@ impl HasTokens for StmtKind { StmtKind::MacCall(mac) => mac.tokens.as_ref(), } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { match self { StmtKind::Let(local) => Some(&mut local.tokens), StmtKind::Item(item) => item.tokens_mut(), @@ -168,16 +168,16 @@ impl HasTokens for StmtKind { } impl HasTokens for Stmt { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { self.kind.tokens() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.kind.tokens_mut() } } impl HasTokens for Attribute { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { match &self.kind { AttrKind::Normal(normal) => normal.tokens.as_ref(), kind @ AttrKind::DocComment(..) => { @@ -185,7 +185,7 @@ impl HasTokens for Attribute { } } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { Some(match &mut self.kind { AttrKind::Normal(normal) => &mut normal.tokens, kind @ AttrKind::DocComment(..) => { @@ -196,7 +196,7 @@ impl HasTokens for Attribute { } impl HasTokens for Nonterminal { - fn tokens(&self) -> Option<&LazyAttrTokenStream> { + fn tokens(&self) -> Option<&AttrTokenStream> { match self { Nonterminal::NtItem(item) => item.tokens(), Nonterminal::NtStmt(stmt) => stmt.tokens(), @@ -209,7 +209,7 @@ impl HasTokens for Nonterminal { Nonterminal::NtBlock(block) => block.tokens(), } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { match self { Nonterminal::NtItem(item) => item.tokens_mut(), Nonterminal::NtStmt(stmt) => stmt.tokens_mut(), diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 088ae9ba44102..c0f444aab95be 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -8,8 +8,8 @@ use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr}; use crate::ast::{Path, PathSegment, DUMMY_NODE_ID}; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, Token}; +use crate::tokenstream::{AttrTokenStream, TokenStream}; use crate::tokenstream::{DelimSpan, Spacing, TokenTree}; -use crate::tokenstream::{LazyAttrTokenStream, TokenStream}; use crate::util::comments; use crate::util::literal::escape_string_symbol; use rustc_index::bit_set::GrowableBitSet; @@ -210,7 +210,6 @@ impl Attribute { .tokens .as_ref() .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) - .to_attr_token_stream() .to_token_trees(), ), &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone( @@ -586,7 +585,7 @@ pub fn mk_attr( pub fn mk_attr_from_item( g: &AttrIdGenerator, item: AttrItem, - tokens: Option, + tokens: Option, style: AttrStyle, span: Span, ) -> Attribute { diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index cbf21317f1a77..e2e9e98de1aa9 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -531,7 +531,7 @@ pub fn noop_visit_ty(ty: &mut P, vis: &mut T) { fields.flat_map_in_place(|field| vis.flat_map_field_def(field)); } } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } @@ -566,7 +566,7 @@ fn noop_visit_path(Path { segments, span, tokens }: &mut Path, vi vis.visit_ident(ident); visit_opt(args, |args| vis.visit_generic_args(args)); } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } @@ -633,7 +633,7 @@ fn noop_visit_local(local: &mut P, vis: &mut T) { vis.visit_block(els); } } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); visit_opt(colon_sp, |sp| vis.visit_span(sp)); vis.visit_span(span); } @@ -648,8 +648,8 @@ fn noop_visit_attribute(attr: &mut Attribute, vis: &mut T) { } = &mut **normal; vis.visit_path(path); visit_attr_args(args, vis); - visit_lazy_tts(tokens, vis); - visit_lazy_tts(attr_tokens, vis); + visit_opt_attr_tts(tokens, vis); + visit_opt_attr_tts(attr_tokens, vis); } AttrKind::DocComment(_kind, _sym) => {} } @@ -706,7 +706,7 @@ fn visit_attr_tt(tt: &mut AttrTokenTree, vis: &mut T) { } AttrTokenTree::AttrsTarget(AttrsTarget { attrs, tokens }) => { visit_attrs(attrs, vis); - visit_lazy_tts_opt_mut(Some(tokens), vis); + visit_attr_tts(tokens, vis); } } } @@ -739,20 +739,14 @@ fn visit_attr_tts(AttrTokenStream(tts): &mut AttrTokenStream, vis } } -fn visit_lazy_tts_opt_mut(lazy_tts: Option<&mut LazyAttrTokenStream>, vis: &mut T) { - if T::VISIT_TOKENS { - if let Some(lazy_tts) = lazy_tts { - let mut tts = lazy_tts.to_attr_token_stream(); - visit_attr_tts(&mut tts, vis); - *lazy_tts = LazyAttrTokenStream::new(tts); - } +fn visit_opt_attr_tts(attr_tts: &mut Option, vis: &mut T) { + if T::VISIT_TOKENS + && let Some(mut attr_tts) = attr_tts.as_mut() + { + visit_attr_tts(&mut attr_tts, vis) } } -fn visit_lazy_tts(lazy_tts: &mut Option, vis: &mut T) { - visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis); -} - /// Applies ident visitor if it's an ident; applies other visits to interpolated nodes. /// In practice the ident part is not actually used by specific visitors right now, /// but there's a test below checking that it works. @@ -829,7 +823,7 @@ fn visit_nonterminal(nt: &mut token::Nonterminal, vis: &mut T) { let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut(); vis.visit_path(path); visit_attr_args(args, vis); - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); } token::NtPath(path) => vis.visit_path(path), token::NtVis(visib) => vis.visit_vis(visib), @@ -1067,7 +1061,7 @@ pub fn noop_visit_block(block: &mut P, vis: &mut T) { let Block { id, stmts, rules: _, span, tokens, could_be_bare_literal: _ } = block.deref_mut(); vis.visit_id(id); stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt)); - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } @@ -1290,7 +1284,7 @@ pub fn noop_flat_map_item( visitor.visit_vis(vis); visitor.visit_ident(ident); kind.noop_visit(visitor); - visit_lazy_tts(tokens, visitor); + visit_opt_attr_tts(tokens, visitor); visitor.visit_span(span); smallvec![item] } @@ -1365,7 +1359,7 @@ pub fn noop_visit_pat(pat: &mut P, vis: &mut T) { PatKind::Paren(inner) => vis.visit_pat(inner), PatKind::MacCall(mac) => vis.visit_mac_call(mac), } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } @@ -1611,7 +1605,7 @@ pub fn noop_visit_expr( ExprKind::Err(_guar) => {} ExprKind::Dummy => {} } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } @@ -1655,7 +1649,7 @@ fn noop_flat_map_stmt_kind(kind: StmtKind, vis: &mut T) -> SmallV let MacCallStmt { mac: mac_, style: _, attrs, tokens } = mac.deref_mut(); visit_attrs(attrs, vis); vis.visit_mac_call(mac_); - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); smallvec![StmtKind::MacCall(mac)] } } @@ -1670,7 +1664,7 @@ fn noop_visit_vis(visibility: &mut Visibility, vis: &mut T) { vis.visit_path(path); } } - visit_lazy_tts(tokens, vis); + visit_opt_attr_tts(tokens, vis); vis.visit_span(span); } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index ee068f19332a5..69df5487819ba 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -21,8 +21,7 @@ use crate::AttrVec; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{self, Lrc}; use rustc_macros::{Decodable, Encodable, HashStable_Generic}; -use rustc_serialize::{Decodable, Encodable}; -use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP}; +use rustc_span::{sym, Span, Symbol, DUMMY_SP}; use std::borrow::Cow; use std::{cmp, fmt, iter}; @@ -106,56 +105,6 @@ where } } -pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync { - fn to_attr_token_stream(&self) -> AttrTokenStream; -} - -impl ToAttrTokenStream for AttrTokenStream { - fn to_attr_token_stream(&self) -> AttrTokenStream { - self.clone() - } -} - -/// A lazy version of [`TokenStream`], which defers creation -/// of an actual `TokenStream` until it is needed. -/// `Box` is here only to reduce the structure size. -#[derive(Clone)] -pub struct LazyAttrTokenStream(Lrc>); - -impl LazyAttrTokenStream { - pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream { - LazyAttrTokenStream(Lrc::new(Box::new(inner))) - } - - pub fn to_attr_token_stream(&self) -> AttrTokenStream { - self.0.to_attr_token_stream() - } -} - -impl fmt::Debug for LazyAttrTokenStream { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream()) - } -} - -impl Encodable for LazyAttrTokenStream { - fn encode(&self, _s: &mut S) { - panic!("Attempted to encode LazyAttrTokenStream"); - } -} - -impl Decodable for LazyAttrTokenStream { - fn decode(_d: &mut D) -> Self { - panic!("Attempted to decode LazyAttrTokenStream"); - } -} - -impl HashStable for LazyAttrTokenStream { - fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { - panic!("Attempted to compute stable hash for LazyAttrTokenStream"); - } -} - /// An `AttrTokenStream` is similar to a `TokenStream`, but with extra /// information about the tokens for attribute targets. This is used /// during expansion to perform early cfg-expansion, and to process attributes @@ -163,6 +112,12 @@ impl HashStable for LazyAttrTokenStream { #[derive(Clone, Debug, Default, Encodable, Decodable)] pub struct AttrTokenStream(pub Lrc>); +impl HashStable for AttrTokenStream { + fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { + panic!("Attempted to compute stable hash for AttrTokenStream"); + } +} + /// Like `TokenTree`, but for `AttrTokenStream`. #[derive(Clone, Debug, Encodable, Decodable)] pub enum AttrTokenTree { @@ -205,7 +160,7 @@ impl AttrTokenStream { .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); let (outer_attrs, inner_attrs) = target.attrs.split_at(idx); - let mut target_tokens = target.tokens.to_attr_token_stream().to_token_trees(); + let mut target_tokens = target.tokens.to_token_trees(); if !inner_attrs.is_empty() { let mut found = false; // Check the last two trees (to account for a trailing semi) @@ -268,7 +223,7 @@ pub struct AttrsTarget { pub attrs: AttrVec, /// The underlying tokens for the attribute target that `attrs` /// are applied to - pub tokens: LazyAttrTokenStream, + pub tokens: AttrTokenStream, } /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. @@ -442,7 +397,7 @@ impl TokenStream { }; let attrs = node.attrs(); let attr_stream = if attrs.is_empty() { - tokens.to_attr_token_stream() + tokens.clone() } else { let target = AttrsTarget { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; @@ -764,8 +719,7 @@ mod size_asserts { // tidy-alphabetical-start static_assert_size!(AttrTokenStream, 8); static_assert_size!(AttrTokenTree, 32); - static_assert_size!(LazyAttrTokenStream, 8); - static_assert_size!(Option, 8); // must be small, used in many AST nodes + static_assert_size!(Option, 8); // must be small, used in many AST nodes static_assert_size!(TokenStream, 8); static_assert_size!(TokenTree, 32); // tidy-alphabetical-end diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 40e16b4511575..b06d09e47f22e 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -6,8 +6,8 @@ use crate::errors::{ }; use rustc_ast::ptr::P; use rustc_ast::token::{Delimiter, Token, TokenKind}; +use rustc_ast::tokenstream::TokenTree; use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, Spacing}; -use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree}; use rustc_ast::NodeId; use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem}; use rustc_attr as attr; @@ -159,8 +159,7 @@ impl<'a> StripUnconfigured<'a> { fn try_configure_tokens(&self, node: &mut T) { if self.config_tokens { if let Some(Some(tokens)) = node.tokens_mut() { - let attr_stream = tokens.to_attr_token_stream(); - *tokens = LazyAttrTokenStream::new(self.configure_tokens(&attr_stream)); + *tokens = self.configure_tokens(tokens); } } } @@ -190,9 +189,7 @@ impl<'a> StripUnconfigured<'a> { target.attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr)); if self.in_cfg(&target.attrs) { - target.tokens = LazyAttrTokenStream::new( - self.configure_tokens(&target.tokens.to_attr_token_stream()), - ); + target.tokens = self.configure_tokens(&target.tokens); Some(AttrTokenTree::AttrsTarget(target)) } else { None @@ -315,10 +312,7 @@ impl<'a> StripUnconfigured<'a> { DelimSpan::from_single(pound_token.span), DelimSpacing::new(Spacing::JointHidden, Spacing::Alone), Delimiter::Bracket, - item.tokens - .as_ref() - .unwrap_or_else(|| panic!("Missing tokens for {item:?}")) - .to_attr_token_stream(), + item.tokens.as_ref().unwrap_or_else(|| panic!("Missing tokens for {item:?}")).clone(), ); let trees = if attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` @@ -335,7 +329,7 @@ impl<'a> StripUnconfigured<'a> { } else { vec![AttrTokenTree::Token(pound_token, Spacing::JointHidden), bracket_group] }; - let tokens = Some(LazyAttrTokenStream::new(AttrTokenStream::new(trees))); + let tokens = Some(AttrTokenStream::new(trees)); let attr = attr::mk_attr_from_item( &self.sess.psess.attr_id_generator, item, diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index a8fe35f45b31e..94a3349eb37e1 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -305,7 +305,7 @@ impl<'a> Parser<'a> { if let Some(attr) = attr { let end_pos = self.num_bump_calls; // If we are currently capturing tokens, mark the location of this inner attribute. - // If capturing ends up creating a `LazyAttrTokenStream`, we will include + // If capturing ends up creating an `AttrTokenStream`, we will include // this replace range with it, removing the inner attribute from the final // `AttrTokenStream`. Inner attributes are stored in the parsed AST note. // During macro expansion, they are selectively inserted back into the diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 38f18022e3c58..573f92112ff2e 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -1,7 +1,7 @@ -use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken}; +use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TrailingToken}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, AttrsTarget, DelimSpacing}; -use rustc_ast::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, ToAttrTokenStream}; +use rustc_ast::tokenstream::{DelimSpan, Spacing}; use rustc_ast::{self as ast}; use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens}; use rustc_errors::PResult; @@ -76,102 +76,10 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool { }) } -// Produces a `TokenStream` on-demand. Using `cursor_snapshot` -// and `num_calls`, we can reconstruct the `TokenStream` seen -// by the callback. This allows us to avoid producing a `TokenStream` -// if it is never needed - for example, a captured `macro_rules!` -// argument that is never passed to a proc macro. -// In practice token stream creation happens rarely compared to -// calls to `collect_tokens` (see some statistics in #78736), -// so we are doing as little up-front work as possible. -// -// This also makes `Parser` very cheap to clone, since -// there is no intermediate collection buffer to clone. -struct LazyAttrTokenStreamImpl { - start_token: (Token, Spacing), - cursor_snapshot: TokenCursor, - num_calls: u32, - break_last_token: bool, - replace_ranges: Box<[ReplaceRange]>, -} - -impl ToAttrTokenStream for LazyAttrTokenStreamImpl { - fn to_attr_token_stream(&self) -> AttrTokenStream { - // The token produced by the final call to `{,inlined_}next` was not - // actually consumed by the callback. The combination of chaining the - // initial token and using `take` produces the desired result - we - // produce an empty `TokenStream` if no calls were made, and omit the - // final token otherwise. - let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain(iter::repeat_with(|| { - let token = cursor_snapshot.next(); - (FlatToken::Token(token.0), token.1) - })) - .take(self.num_calls as usize); - - if self.replace_ranges.is_empty() { - make_attr_token_stream(tokens, self.break_last_token) - } else { - let mut tokens: Vec<_> = tokens.collect(); - let mut replace_ranges = self.replace_ranges.to_vec(); - replace_ranges.sort_by_key(|(range, _)| range.start); - - #[cfg(debug_assertions)] - { - for [(range, tokens), (next_range, next_tokens)] in replace_ranges.array_windows() { - assert!( - range.end <= next_range.start || range.end >= next_range.end, - "Replace ranges should either be disjoint or nested: ({:?}, {:?}) ({:?}, {:?})", - range, - tokens, - next_range, - next_tokens, - ); - } - } - - // Process the replace ranges, starting from the highest start - // position and working our way back. If have tokens like: - // - // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` - // - // Then we will generate replace ranges for both - // the `#[cfg(FALSE)] field: bool` and the entire - // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` - // - // By starting processing from the replace range with the greatest - // start position, we ensure that any replace range which encloses - // another replace range will capture the *replaced* tokens for the inner - // range, not the original tokens. - for (range, target) in replace_ranges.into_iter().rev() { - assert!(!range.is_empty(), "Cannot replace an empty range: {range:?}"); - - // Replace the tokens in range with zero or one `FlatToken::AttrsTarget`s, plus - // enough `FlatToken::Empty`s to fill up the rest of the range. This keeps the - // total length of `tokens` constant throughout the replacement process, allowing - // us to use all of the `ReplaceRanges` entries without adjusting indices. - let target_len = target.is_some() as usize; - tokens.splice( - (range.start as usize)..(range.end as usize), - target - .into_iter() - .map(|target| (FlatToken::AttrsTarget(target), Spacing::Alone)) - .chain( - iter::repeat((FlatToken::Empty, Spacing::Alone)) - .take(range.len() - target_len), - ), - ); - } - make_attr_token_stream(tokens.into_iter(), self.break_last_token) - } - } -} - impl<'a> Parser<'a> { /// Records all tokens consumed by the provided callback, /// including the current token. These tokens are collected - /// into a `LazyAttrTokenStream`, and returned along with the result + /// into an `AttrTokenStream`, and returned along with the result /// of the callback. /// /// The `attrs` passed in are in `AttrWrapper` form, which is opaque. The @@ -213,7 +121,7 @@ impl<'a> Parser<'a> { } let start_token = (self.token.clone(), self.token_spacing); - let cursor_snapshot = self.token_cursor.clone(); + let mut cursor_snapshot = self.token_cursor.clone(); let start_pos = self.num_bump_calls; let has_outer_attrs = !attrs.attrs.is_empty(); let replace_ranges_start = self.capture_state.replace_ranges.len(); @@ -295,42 +203,87 @@ impl<'a> Parser<'a> { + captured_trailing as u32 // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then // extend the range of captured tokens to include it, since the parser was not actually - // bumped past it. When the `LazyAttrTokenStream` gets converted into an - // `AttrTokenStream`, we will create the proper token. + // bumped past it. We will create the proper token below. + self.break_last_token as u32; let num_calls = end_pos - start_pos; + // The token produced by the final call to `{,inlined_}next` was not + // actually consumed by the callback. The combination of chaining the + // initial token and using `take` produces the desired result - we + // produce an empty `TokenStream` if no calls were made, and omit the + // final token otherwise. + let tokens_iter = iter::once(FlatToken::Token(start_token)) + .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next()))) + .take(num_calls as usize); + // If we have no attributes, then we will never need to // use any replace ranges. - let replace_ranges: Box<[ReplaceRange]> = if ret.attrs().is_empty() && !self.capture_cfg { - Box::new([]) + let tokens = if ret.attrs().is_empty() && !self.capture_cfg { + make_attr_token_stream(tokens_iter, self.break_last_token) } else { + let mut tokens: Vec<_> = tokens_iter.collect(); + // Grab any replace ranges that occur *inside* the current AST node. - // We will perform the actual replacement when we convert the `LazyAttrTokenStream` - // to an `AttrTokenStream`. - self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] + // We will perform the actual replacement below. + let mut replace_ranges: Vec = self.capture_state.replace_ranges + [replace_ranges_start..replace_ranges_end] .iter() .cloned() .chain(inner_attr_replace_ranges.iter().cloned()) .map(|(range, data)| ((range.start - start_pos)..(range.end - start_pos), data)) - .collect() - }; + .collect(); + replace_ranges.sort_by_key(|(range, _)| range.start); - let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl { - start_token, - num_calls, - cursor_snapshot, - break_last_token: self.break_last_token, - replace_ranges, - }); - - // If we support tokens at all - if let Some(target_tokens) = ret.tokens_mut() { - if target_tokens.is_none() { - // Store our newly captured tokens into the AST node. - *target_tokens = Some(tokens.clone()); + #[cfg(debug_assertions)] + { + for [(range, tokens), (next_range, next_tokens)] in replace_ranges.array_windows() { + assert!( + range.end <= next_range.start || range.end >= next_range.end, + "Replace ranges should either be disjoint or nested: ({:?}, {:?}) ({:?}, {:?})", + range, + tokens, + next_range, + next_tokens, + ); + } } + + // Process the replace ranges, starting from the highest start + // position and working our way back. If have tokens like: + // + // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` + // + // Then we will generate replace ranges for both + // the `#[cfg(FALSE)] field: bool` and the entire + // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }` + // + // By starting processing from the replace range with the greatest + // start position, we ensure that any replace range which encloses + // another replace range will capture the *replaced* tokens for the inner + // range, not the original tokens. + for (range, target) in replace_ranges.into_iter().rev() { + assert!(!range.is_empty(), "Cannot replace an empty range: {range:?}"); + + // Replace the tokens in range with zero or one `FlatToken::AttrsTarget`s, plus + // enough `FlatToken::Empty`s to fill up the rest of the range. This keeps the + // total length of `tokens` constant throughout the replacement process, allowing + // us to use all of the `ReplaceRanges` entries without adjusting indices. + let target_len = target.is_some() as usize; + tokens.splice( + (range.start as usize)..(range.end as usize), + target + .into_iter() + .map(|target| FlatToken::AttrsTarget(target)) + .chain(iter::repeat(FlatToken::Empty).take(range.len() - target_len)), + ); + } + make_attr_token_stream(tokens.into_iter(), self.break_last_token) + }; + + // If we support tokens and don't already have them, store the newly captured tokens. + if let Some(target_tokens @ None) = ret.tokens_mut() { + *target_tokens = Some(tokens.clone()); } let final_attrs = ret.attrs(); @@ -370,7 +323,7 @@ impl<'a> Parser<'a> { /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// close delims. fn make_attr_token_stream( - mut iter: impl Iterator, + iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { #[derive(Debug)] @@ -379,19 +332,19 @@ fn make_attr_token_stream( open_delim_sp: Option<(Delimiter, Span, Spacing)>, inner: Vec, } - let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; - let mut token_and_spacing = iter.next(); - while let Some((token, spacing)) = token_and_spacing { - match token { - FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { - stack - .push(FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }); + // The stack always has at least one element. Storing it separately makes for shorter code. + let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; + let mut stack_rest = vec![]; + for flat_token in iter { + match flat_token { + FlatToken::Token((Token { kind: TokenKind::OpenDelim(delim), span }, spacing)) => { + stack_rest.push(mem::replace( + &mut stack_top, + FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, + )); } - FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { - let frame_data = stack - .pop() - .unwrap_or_else(|| panic!("Token stack was empty for token: {token:?}")); - + FlatToken::Token((Token { kind: TokenKind::CloseDelim(delim), span }, spacing)) => { + let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); assert_eq!( open_delim, delim, @@ -401,29 +354,20 @@ fn make_attr_token_stream( let dspacing = DelimSpacing::new(open_spacing, spacing); let stream = AttrTokenStream::new(frame_data.inner); let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); - stack - .last_mut() - .unwrap_or_else(|| panic!("Bottom token frame is missing for token: {token:?}")) - .inner - .push(delimited); + stack_top.inner.push(delimited); + } + FlatToken::Token((token, spacing)) => { + stack_top.inner.push(AttrTokenTree::Token(token, spacing)) + } + FlatToken::AttrsTarget(target) => { + stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) } - FlatToken::Token(token) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::Token(token, spacing)), - FlatToken::AttrsTarget(target) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::AttrsTarget(target)), FlatToken::Empty => {} } - token_and_spacing = iter.next(); } - let mut final_buf = stack.pop().expect("Missing final buf!"); + if break_last_token { - let last_token = final_buf.inner.pop().unwrap(); + let last_token = stack_top.inner.pop().unwrap(); if let AttrTokenTree::Token(last_token, spacing) = last_token { let unglued_first = last_token.kind.break_two_token_op().unwrap().0; @@ -431,14 +375,14 @@ fn make_attr_token_stream( let mut first_span = last_token.span.shrink_to_lo(); first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); - final_buf + stack_top .inner .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing)); } else { panic!("Unexpected last token {last_token:?}") } } - AttrTokenStream::new(final_buf.inner) + AttrTokenStream::new(stack_top.inner) } // Some types are used a lot. Make sure they don't unintentionally get bigger. @@ -448,6 +392,5 @@ mod size_asserts { use rustc_data_structures::static_assert_size; // tidy-alphabetical-start static_assert_size!(AttrWrapper, 16); - static_assert_size!(LazyAttrTokenStreamImpl, 96); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 81d5f0fca0ec9..3dbcf2f318acf 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -2371,7 +2371,6 @@ impl<'a> Parser<'a> { let tokens = nt.tokens(); labels.push(nt.clone()); if let Some(tokens) = tokens - && let tokens = tokens.to_attr_token_stream() && let tokens = tokens.0.deref() && let [AttrTokenTree::Token(token, _)] = &tokens[..] { diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 45ca267fe5d1e..a532f34910386 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -167,7 +167,7 @@ pub struct Parser<'a> { // token. For example, capturing the `Vec` in `Option>` // requires us to unglue the trailing `>>` token. The `break_last_token` // field is used to track this token. It gets appended to the captured - // stream when we evaluate a `LazyAttrTokenStream`. + // stream when we create an `AttrTokenStream`. break_last_token: bool, /// This field is used to keep track of how many left angle brackets we have seen. This is /// required in order to detect extra leading left angle brackets (`<` characters) and error @@ -1598,16 +1598,15 @@ pub(crate) fn make_unclosed_delims_error( Some(err) } -/// A helper struct used when building an `AttrTokenStream` from -/// a `LazyAttrTokenStream`. Both delimiter and non-delimited tokens -/// are stored as `FlatToken::Token`. A vector of `FlatToken`s -/// is then 'parsed' to build up an `AttrTokenStream` with nested -/// `AttrTokenTree::Delimited` tokens. +/// A helper struct used when building an `AttrTokenStream`. Both delimiter and +/// non-delimited tokens are stored as `FlatToken::Token`. A sequence of +/// `FlatToken`s is then "parsed" to build up tree-structured +/// `AttrTokenStream`. #[derive(Debug, Clone)] enum FlatToken { /// A token - this holds both delimiter (e.g. '{' and '}') /// and non-delimiter tokens - Token(Token), + Token((Token, Spacing)), /// Holds the `AttrsTarget` for an AST node. The `AttrsTarget` is inserted /// directly into the constructed `AttrTokenStream` as an /// `AttrTokenTree::AttrsTarget`.