Subtree update of rust-analyzer

src/tools/rust-analyzer/.github/workflows/ci.yaml

@@ -6,6 +6,10 @@ on:
   pull_request:
   merge_group:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
   CARGO_INCREMENTAL: 0
   CARGO_NET_RETRY: 10
@@ -103,7 +107,7 @@ jobs:
       - name: Run analysis-stats on the rust standard libraries
         if: matrix.os == 'ubuntu-latest'
         env:
-            RUSTC_BOOTSTRAP: 1
+          RUSTC_BOOTSTRAP: 1
         run: target/${{ matrix.target }}/debug/rust-analyzer analysis-stats --with-deps --no-sysroot --no-test $(rustc --print sysroot)/lib/rustlib/src/rust/library/
 
       - name: clippy

src/tools/rust-analyzer/Cargo.lock

@@ -1375,7 +1375,6 @@ dependencies = [
  "memmap2",
  "object 0.33.0",
  "paths",
- "proc-macro-api",
  "proc-macro-test",
  "ra-ap-rustc_lexer",
  "span",
@@ -1390,6 +1389,7 @@ version = "0.0.0"
 dependencies = [
  "proc-macro-api",
  "proc-macro-srv",
+ "tt",
 ]
 
 [[package]]

src/tools/rust-analyzer/Cargo.toml

@@ -79,6 +79,7 @@ span = { path = "./crates/span", version = "0.0.0" }
 stdx = { path = "./crates/stdx", version = "0.0.0" }
 syntax = { path = "./crates/syntax", version = "0.0.0" }
 syntax-bridge = { path = "./crates/syntax-bridge", version = "0.0.0" }
+test-utils = { path = "./crates/test-utils", version = "0.0.0" }
 toolchain = { path = "./crates/toolchain", version = "0.0.0" }
 tt = { path = "./crates/tt", version = "0.0.0" }
 vfs-notify = { path = "./crates/vfs-notify", version = "0.0.0" }
@@ -93,7 +94,6 @@ ra-ap-rustc_pattern_analysis = { version = "0.87", default-features = false }
 
 # local crates that aren't published to crates.io. These should not have versions.
 test-fixture = { path = "./crates/test-fixture" }
-test-utils = { path = "./crates/test-utils" }
 
 # In-tree crates that are published separately and follow semver. See lib/README.md
 line-index = { version = "0.1.2" }
@@ -203,6 +203,13 @@ new_ret_no_self = "allow"
 useless_asref = "allow"
 # Has false positives
 assigning_clones = "allow"
+# Does not work with macros
+vec_init_then_push = "allow"
+# Our tests have a lot of these
+literal_string_with_formatting_args = "allow"
+# This lint has been empowered but now also triggers on cases where its invalid to do so
+# due to it ignoring move analysis
+unnecessary_map_or = "allow"
 
 ## Following lints should be tackled at some point
 too_many_arguments = "allow"

src/tools/rust-analyzer/clippy.toml

@@ -3,3 +3,7 @@ disallowed-types = [
     { path = "std::collections::HashSet", reason = "use FxHashSet" },
     { path = "std::collections::hash_map::RandomState", reason = "use BuildHasherDefault<FxHasher>"}
 ]
+
+disallowed-methods = [
+    { path = "std::process::Command::new", reason = "use `toolchain::command` instead as it forces the choice of a working directory" },
+]

src/tools/rust-analyzer/crates/base-db/src/input.rs

@@ -490,21 +490,25 @@ impl CrateGraph {
         }
     }
 
-    pub fn sort_deps(&mut self) {
-        self.arena
-            .iter_mut()
-            .for_each(|(_, data)| data.dependencies.sort_by_key(|dep| dep.crate_id));
-    }
-
     /// Extends this crate graph by adding a complete second crate
     /// graph and adjust the ids in the [`ProcMacroPaths`] accordingly.
     ///
+    /// This will deduplicate the crates of the graph where possible.
+    /// Furthermore dependencies are sorted by crate id to make deduplication easier.
+    ///
     /// Returns a map mapping `other`'s IDs to the new IDs in `self`.
     pub fn extend(
         &mut self,
         mut other: CrateGraph,
         proc_macros: &mut ProcMacroPaths,
     ) -> FxHashMap<CrateId, CrateId> {
+        // Sorting here is a bit pointless because the input is likely already sorted.
+        // However, the overhead is small and it makes the `extend` method harder to misuse.
+        self.arena
+            .iter_mut()
+            .for_each(|(_, data)| data.dependencies.sort_by_key(|dep| dep.crate_id));
+
+        let m = self.len();
         let topo = other.crates_in_topological_order();
         let mut id_map: FxHashMap<CrateId, CrateId> = FxHashMap::default();
         for topo in topo {
@@ -513,7 +517,8 @@ impl CrateGraph {
             crate_data.dependencies.iter_mut().for_each(|dep| dep.crate_id = id_map[&dep.crate_id]);
             crate_data.dependencies.sort_by_key(|dep| dep.crate_id);
 
-            let new_id = self.arena.alloc(crate_data.clone());
+            let find = self.arena.iter().take(m).find_map(|(k, v)| (v == crate_data).then_some(k));
+            let new_id = find.unwrap_or_else(|| self.arena.alloc(crate_data.clone()));
             id_map.insert(topo, new_id);
         }
 

src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs

@@ -45,8 +45,8 @@ impl From<CfgAtom> for CfgExpr {
 
 impl CfgExpr {
     #[cfg(feature = "tt")]
-    pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
-        next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
+    pub fn parse<S: Copy>(tt: &tt::TopSubtree<S>) -> CfgExpr {
+        next_cfg_expr(&mut tt.iter()).unwrap_or(CfgExpr::Invalid)
     }
 
     /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -66,19 +66,19 @@ impl CfgExpr {
 }
 
 #[cfg(feature = "tt")]
-fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
+fn next_cfg_expr<S: Copy>(it: &mut tt::iter::TtIter<'_, S>) -> Option<CfgExpr> {
     use intern::sym;
+    use tt::iter::TtElement;
 
     let name = match it.next() {
         None => return None,
-        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
+        Some(TtElement::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
         Some(_) => return Some(CfgExpr::Invalid),
     };
 
-    // Peek
-    let ret = match it.as_slice().first() {
-        Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
-            match it.as_slice().get(1) {
+    let ret = match it.peek() {
+        Some(TtElement::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
+            match it.remaining().flat_tokens().get(1) {
                 Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
                     it.next();
                     it.next();
@@ -87,9 +87,8 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
                 _ => return Some(CfgExpr::Invalid),
             }
         }
-        Some(tt::TokenTree::Subtree(subtree)) => {
+        Some(TtElement::Subtree(_, mut sub_it)) => {
             it.next();
-            let mut sub_it = subtree.token_trees.iter();
             let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it));
             match name {
                 s if s == sym::all => CfgExpr::All(subs.collect()),
@@ -104,7 +103,7 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
     };
 
     // Eat comma separator
-    if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
+    if let Some(TtElement::Leaf(tt::Leaf::Punct(punct))) = it.peek() {
         if punct.char == ',' {
             it.next();
         }

src/tools/rust-analyzer/crates/edition/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "edition"
 version = "0.0.0"
+description = "Rust edition support crate for rust-analyzer."
 rust-version.workspace = true
 edition.workspace = true
 license.workspace = true

src/tools/rust-analyzer/crates/hir-def/src/attr.rs

@@ -1,6 +1,6 @@
 //! A higher level attributes based on TokenTree, with also some shortcuts.
 
-use std::{borrow::Cow, hash::Hash, ops, slice};
+use std::{borrow::Cow, hash::Hash, ops};
 
 use base_db::CrateId;
 use cfg::{CfgExpr, CfgOptions};
@@ -17,6 +17,7 @@ use syntax::{
     AstPtr,
 };
 use triomphe::Arc;
+use tt::iter::{TtElement, TtIter};
 
 use crate::{
     db::DefDatabase,
@@ -154,15 +155,15 @@ impl Attrs {
 
     pub fn has_doc_hidden(&self) -> bool {
         self.by_key(&sym::doc).tt_values().any(|tt| {
-            tt.delimiter.kind == DelimiterKind::Parenthesis &&
-                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
+            tt.top_subtree().delimiter.kind == DelimiterKind::Parenthesis &&
+                matches!(tt.token_trees().flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
         })
     }
 
     pub fn has_doc_notable_trait(&self) -> bool {
         self.by_key(&sym::doc).tt_values().any(|tt| {
-            tt.delimiter.kind == DelimiterKind::Parenthesis &&
-                matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
+            tt.top_subtree().delimiter.kind == DelimiterKind::Parenthesis &&
+                matches!(tt.token_trees().flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
         })
     }
 
@@ -245,8 +246,8 @@ impl From<DocAtom> for DocExpr {
 }
 
 impl DocExpr {
-    fn parse<S>(tt: &tt::Subtree<S>) -> DocExpr {
-        next_doc_expr(&mut tt.token_trees.iter()).unwrap_or(DocExpr::Invalid)
+    fn parse<S: Copy>(tt: &tt::TopSubtree<S>) -> DocExpr {
+        next_doc_expr(tt.iter()).unwrap_or(DocExpr::Invalid)
     }
 
     pub fn aliases(&self) -> &[Symbol] {
@@ -260,62 +261,47 @@ impl DocExpr {
     }
 }
 
-fn next_doc_expr<S>(it: &mut slice::Iter<'_, tt::TokenTree<S>>) -> Option<DocExpr> {
+fn next_doc_expr<S: Copy>(mut it: TtIter<'_, S>) -> Option<DocExpr> {
     let name = match it.next() {
         None => return None,
-        Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
+        Some(TtElement::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
         Some(_) => return Some(DocExpr::Invalid),
     };
 
     // Peek
-    let ret = match it.as_slice().first() {
-        Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
-            match it.as_slice().get(1) {
-                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+    let ret = match it.peek() {
+        Some(TtElement::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
+            it.next();
+            match it.next() {
+                Some(TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
                     symbol: text,
                     kind: tt::LitKind::Str,
                     ..
-                }))) => {
-                    it.next();
-                    it.next();
-                    DocAtom::KeyValue { key: name, value: text.clone() }.into()
-                }
+                }))) => DocAtom::KeyValue { key: name, value: text.clone() }.into(),
                 _ => return Some(DocExpr::Invalid),
             }
         }
-        Some(tt::TokenTree::Subtree(subtree)) => {
+        Some(TtElement::Subtree(_, subtree_iter)) => {
             it.next();
-            let subs = parse_comma_sep(subtree);
+            let subs = parse_comma_sep(subtree_iter);
             match &name {
                 s if *s == sym::alias => DocExpr::Alias(subs),
                 _ => DocExpr::Invalid,
             }
         }
         _ => DocAtom::Flag(name).into(),
     };
-
-    // Eat comma separator
-    if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
-        if punct.char == ',' {
-            it.next();
-        }
-    }
     Some(ret)
 }
 
-fn parse_comma_sep<S>(subtree: &tt::Subtree<S>) -> Vec<Symbol> {
-    subtree
-        .token_trees
-        .iter()
-        .filter_map(|tt| match tt {
-            tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
-                kind: tt::LitKind::Str,
-                symbol,
-                ..
-            })) => Some(symbol.clone()),
-            _ => None,
-        })
-        .collect()
+fn parse_comma_sep<S>(iter: TtIter<'_, S>) -> Vec<Symbol> {
+    iter.filter_map(|tt| match tt {
+        TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
+            kind: tt::LitKind::Str, symbol, ..
+        })) => Some(symbol.clone()),
+        _ => None,
+    })
+    .collect()
 }
 
 impl AttrsWithOwner {
@@ -563,7 +549,7 @@ pub struct AttrQuery<'attr> {
 }
 
 impl<'attr> AttrQuery<'attr> {
-    pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
+    pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::TopSubtree> {
         self.attrs().filter_map(|attr| attr.token_tree_value())
     }
 
@@ -585,7 +571,7 @@ impl<'attr> AttrQuery<'attr> {
 
     pub fn attrs(self) -> impl Iterator<Item = &'attr Attr> + Clone {
         let key = self.key;
-        self.attrs.iter().filter(move |attr| attr.path.as_ident().map_or(false, |s| *s == *key))
+        self.attrs.iter().filter(move |attr| attr.path.as_ident().is_some_and(|s| *s == *key))
     }
 
     /// Find string value for a specific key inside token tree
@@ -596,12 +582,12 @@ impl<'attr> AttrQuery<'attr> {
     /// ```
     pub fn find_string_value_in_tt(self, key: &'attr Symbol) -> Option<&'attr str> {
         self.tt_values().find_map(|tt| {
-            let name = tt.token_trees.iter()
-                .skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if *sym == *key))
+            let name = tt.iter()
+                .skip_while(|tt| !matches!(tt, TtElement::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if *sym == *key))
                 .nth(2);
 
             match name {
-                Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{  symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
+                Some(TtElement::Leaf(tt::Leaf::Literal(tt::Literal{  symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
                 _ => None
             }
         })

src/tools/rust-analyzer/crates/hir-def/src/body/lower.rs

@@ -2216,11 +2216,11 @@ impl ExprCollector<'_> {
         };
         // This needs to match `Flag` in library/core/src/fmt/rt.rs.
         let flags: u32 = ((sign == Some(FormatSign::Plus)) as u32)
-            | ((sign == Some(FormatSign::Minus)) as u32) << 1
-            | (alternate as u32) << 2
-            | (zero_pad as u32) << 3
-            | ((debug_hex == Some(FormatDebugHex::Lower)) as u32) << 4
-            | ((debug_hex == Some(FormatDebugHex::Upper)) as u32) << 5;
+            | (((sign == Some(FormatSign::Minus)) as u32) << 1)
+            | ((alternate as u32) << 2)
+            | ((zero_pad as u32) << 3)
+            | (((debug_hex == Some(FormatDebugHex::Lower)) as u32) << 4)
+            | (((debug_hex == Some(FormatDebugHex::Upper)) as u32) << 5);
         let flags = self.alloc_expr_desugared(Expr::Literal(Literal::Uint(
             flags as u128,
             Some(BuiltinUint::U32),
@@ -2468,7 +2468,7 @@ impl ExprCollector<'_> {
 
 fn comma_follows_token(t: Option<syntax::SyntaxToken>) -> bool {
     (|| syntax::algo::skip_trivia_token(t?.next_token()?, syntax::Direction::Next))()
-        .map_or(false, |it| it.kind() == syntax::T![,])
+        .is_some_and(|it| it.kind() == syntax::T![,])
 }
 
 #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]

src/tools/rust-analyzer/crates/hir-def/src/body/tests.rs

@@ -52,6 +52,7 @@ fn your_stack_belongs_to_me() {
     cov_mark::check!(your_stack_belongs_to_me);
     lower(
         r#"
+#![recursion_limit = "32"]
 macro_rules! n_nuple {
     ($e:tt) => ();
     ($($rest:tt)*) => {{
@@ -68,6 +69,7 @@ fn your_stack_belongs_to_me2() {
     cov_mark::check!(overflow_but_not_me);
     lower(
         r#"
+#![recursion_limit = "32"]
 macro_rules! foo {
     () => {{ foo!(); foo!(); }}
 }
@@ -78,8 +80,6 @@ fn main() { foo!(); }
 
 #[test]
 fn recursion_limit() {
-    cov_mark::check!(your_stack_belongs_to_me);
-
     lower(
         r#"
 #![recursion_limit = "2"]