XXX: Remove LazyAttrTokenStream

Author: nnethercote

compiler/rustc_ast/src/ast.rs

@@ -26,7 +26,7 @@ pub use UnsafeSource::*;
 
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter};
-use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
+use crate::tokenstream::{AttrTokenStream, DelimSpan, TokenStream};
 pub use rustc_ast_ir::{Movability, Mutability};
 use rustc_data_structures::packed::Pu128;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -94,7 +94,7 @@ pub struct Path {
     /// The segments in the path: the things separated by `::`.
     /// Global paths begin with `kw::PathRoot`.
     pub segments: ThinVec<PathSegment>,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl PartialEq<Symbol> for Path {
@@ -544,7 +544,7 @@ pub struct Block {
     /// Distinguishes between `unsafe { ... }` and `{ ... }`.
     pub rules: BlockCheckMode,
     pub span: Span,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
     /// The following *isn't* a parse error, but will cause multiple errors in following stages.
     /// ```compile_fail
     /// let x = {
@@ -563,7 +563,7 @@ pub struct Pat {
     pub id: NodeId,
     pub kind: PatKind,
     pub span: Span,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl Pat {
@@ -1002,7 +1002,7 @@ impl Stmt {
     /// a trailing semicolon.
     ///
     /// This only modifies the parsed AST struct, not the attached
-    /// `LazyAttrTokenStream`. The parser is responsible for calling
+    /// `AttrTokenStream`. The parser is responsible for calling
     /// `ToAttrTokenStream::add_trailing_semi` when there is actually
     /// a semicolon in the tokenstream.
     pub fn add_trailing_semicolon(mut self) -> Self {
@@ -1050,7 +1050,7 @@ pub struct MacCallStmt {
     pub mac: P<MacCall>,
     pub style: MacStmtStyle,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@@ -1076,7 +1076,7 @@ pub struct Local {
     pub span: Span,
     pub colon_sp: Option<Span>,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -1175,7 +1175,7 @@ pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl Expr {
@@ -2102,7 +2102,7 @@ pub struct Ty {
     pub id: NodeId,
     pub kind: TyKind,
     pub span: Span,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl Clone for Ty {
@@ -2831,7 +2831,7 @@ pub enum AttrKind {
 pub struct NormalAttr {
     pub item: AttrItem,
     // Tokens for the full attribute, e.g. `#[foo]`, `#![bar]`.
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl NormalAttr {
@@ -2854,7 +2854,7 @@ pub struct AttrItem {
     pub path: Path,
     pub args: AttrArgs,
     // Tokens for the meta item, e.g. just the `foo` within `#[foo]` or `#![foo]`.
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 /// `TraitRef`s appear in impls.
@@ -2894,7 +2894,7 @@ impl PolyTraitRef {
 pub struct Visibility {
     pub kind: VisibilityKind,
     pub span: Span,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2987,7 +2987,7 @@ pub struct Item<K = ItemKind> {
     ///
     /// Note that the tokens here do not include the outer attributes, but will
     /// include inner attributes.
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<AttrTokenStream>,
 }
 
 impl Item {

compiler/rustc_ast/src/ast_traits.rs

@@ -4,7 +4,7 @@
 
 use crate::ptr::P;
 use crate::token::Nonterminal;
-use crate::tokenstream::LazyAttrTokenStream;
+use crate::tokenstream::AttrTokenStream;
 use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
 use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
 use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
@@ -91,18 +91,18 @@ impl<T: AstDeref<Target: HasNodeId>> HasNodeId for T {
 
 /// A trait for AST nodes having (or not having) collected tokens.
 pub trait HasTokens {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream>;
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>>;
+    fn tokens(&self) -> Option<&AttrTokenStream>;
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>>;
 }
 
 macro_rules! impl_has_tokens {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+                fn tokens(&self) -> Option<&AttrTokenStream> {
                     self.tokens.as_ref()
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
                     Some(&mut self.tokens)
                 }
             }
@@ -114,10 +114,10 @@ macro_rules! impl_has_tokens_none {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+                fn tokens(&self) -> Option<&AttrTokenStream> {
                     None
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
                     None
                 }
             }
@@ -129,25 +129,25 @@ impl_has_tokens!(AssocItem, AttrItem, Block, Expr, ForeignItem, Item, Pat, Path,
 impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant);
 
 impl<T: AstDeref<Target: HasTokens>> HasTokens for T {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         self.ast_deref().tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         self.ast_deref_mut().tokens_mut()
     }
 }
 
 impl<T: HasTokens> HasTokens for Option<T> {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         self.as_ref().and_then(|inner| inner.tokens())
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
 impl HasTokens for StmtKind {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         match self {
             StmtKind::Let(local) => local.tokens.as_ref(),
             StmtKind::Item(item) => item.tokens(),
@@ -156,7 +156,7 @@ impl HasTokens for StmtKind {
             StmtKind::MacCall(mac) => mac.tokens.as_ref(),
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         match self {
             StmtKind::Let(local) => Some(&mut local.tokens),
             StmtKind::Item(item) => item.tokens_mut(),
@@ -168,24 +168,24 @@ impl HasTokens for StmtKind {
 }
 
 impl HasTokens for Stmt {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         self.kind.tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         self.kind.tokens_mut()
     }
 }
 
 impl HasTokens for Attribute {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         match &self.kind {
             AttrKind::Normal(normal) => normal.tokens.as_ref(),
             kind @ AttrKind::DocComment(..) => {
                 panic!("Called tokens on doc comment attr {kind:?}")
             }
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         Some(match &mut self.kind {
             AttrKind::Normal(normal) => &mut normal.tokens,
             kind @ AttrKind::DocComment(..) => {
@@ -196,7 +196,7 @@ impl HasTokens for Attribute {
 }
 
 impl HasTokens for Nonterminal {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&AttrTokenStream> {
         match self {
             Nonterminal::NtItem(item) => item.tokens(),
             Nonterminal::NtStmt(stmt) => stmt.tokens(),
@@ -209,7 +209,7 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtBlock(block) => block.tokens(),
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<AttrTokenStream>> {
         match self {
             Nonterminal::NtItem(item) => item.tokens_mut(),
             Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),

compiler/rustc_ast/src/attr/mod.rs

@@ -8,8 +8,8 @@ use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
 use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter, Token};
+use crate::tokenstream::{AttrTokenStream, TokenStream};
 use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
-use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
 use crate::util::comments;
 use crate::util::literal::escape_string_symbol;
 use rustc_index::bit_set::GrowableBitSet;
@@ -210,7 +210,6 @@ impl Attribute {
                     .tokens
                     .as_ref()
                     .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
-                    .to_attr_token_stream()
                     .to_token_trees(),
             ),
             &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
@@ -586,7 +585,7 @@ pub fn mk_attr(
 pub fn mk_attr_from_item(
     g: &AttrIdGenerator,
     item: AttrItem,
-    tokens: Option<LazyAttrTokenStream>,
+    tokens: Option<AttrTokenStream>,
     style: AttrStyle,
     span: Span,
 ) -> Attribute {