Auto merge of #14061 - Veykril:generic-tt, r=Veykril

Make tt generic over the span data

This also fixes up our delimiter representation in tt, it is no longer optional (we use invisible delims in the same way as before, that is still incorrectly) and we now store two spans instead of one.

These changes should help with adjusting our token map. Though this will probably break proc-macros in some ways, will need to test that for now.

Author: bors

crates/base-db/src/fixture.rs

@@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
 use test_utils::{
     extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
 };
-use tt::Subtree;
+use tt::token_id::Subtree;
 use vfs::{file_set::FileSet, VfsPath};
 
 use crate::{
@@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
         _: &Env,
     ) -> Result<Subtree, ProcMacroExpansionError> {
         fn traverse(input: &Subtree) -> Subtree {
-            let mut res = Subtree::default();
-            res.delimiter = input.delimiter;
+            let mut token_trees = vec![];
             for tt in input.token_trees.iter().rev() {
                 let tt = match tt {
                     tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
                     tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
                 };
-                res.token_trees.push(tt);
+                token_trees.push(tt);
             }
-            res
+            Subtree { delimiter: input.delimiter, token_trees }
         }
         Ok(traverse(input))
     }

crates/base-db/src/input.rs

@@ -12,7 +12,7 @@ use cfg::CfgOptions;
 use rustc_hash::FxHashMap;
 use stdx::hash::{NoHashHashMap, NoHashHashSet};
 use syntax::SmolStr;
-use tt::Subtree;
+use tt::token_id::Subtree;
 use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
 
 /// Files are grouped into source roots. A source root is a directory on the

crates/cfg/src/cfg_expr.rs

@@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
 }
 
 impl CfgExpr {
-    pub fn parse(tt: &tt::Subtree) -> CfgExpr {
+    pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
         next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
     }
     /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -85,7 +85,7 @@ impl CfgExpr {
     }
 }
 
-fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> {
+fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
     let name = match it.next() {
         None => return None,
         Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

crates/hir-def/src/adt.rs

@@ -2,6 +2,7 @@
 
 use std::sync::Arc;
 
+use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
 use base_db::CrateId;
 use either::Either;
 use hir_expand::{
@@ -12,7 +13,6 @@ use intern::Interned;
 use la_arena::{Arena, ArenaMap};
 use rustc_abi::{Integer, IntegerType};
 use syntax::ast::{self, HasName, HasVisibility};
-use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
 
 use crate::{
     body::{CfgExpander, LowerCtx},
@@ -82,7 +82,7 @@ fn repr_from_value(
 
 fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
     match tt.delimiter {
-        Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {}
+        Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
         _ => return None,
     }
 

crates/hir-def/src/attr.rs

@@ -16,7 +16,6 @@ use syntax::{
     ast::{self, HasAttrs, IsString},
     AstPtr, AstToken, SmolStr, TextRange, TextSize,
 };
-use tt::Subtree;
 
 use crate::{
     db::DefDatabase,
@@ -234,7 +233,7 @@ impl Attrs {
 
     pub fn has_doc_hidden(&self) -> bool {
         self.by_key("doc").tt_values().any(|tt| {
-            tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) &&
+            tt.delimiter.kind == DelimiterKind::Parenthesis &&
                 matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
         })
     }
@@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
 }
 
 impl<'attr> AttrQuery<'attr> {
-    pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> {
+    pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
         self.attrs().filter_map(|attr| attr.token_tree_value())
     }
 

crates/hir-def/src/data.rs

@@ -142,7 +142,7 @@ impl FunctionData {
     }
 }
 
-fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> {
+fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
     let mut indices = Vec::new();
     for args in tt.token_trees.chunks(2) {
         match &args[0] {

crates/hir-def/src/lib.rs

@@ -79,6 +79,8 @@ use nameres::DefMap;
 use stdx::impl_from;
 use syntax::ast;
 
+use ::tt::token_id as tt;
+
 use crate::{
     adt::VariantData,
     builtin_type::BuiltinType,
@@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
     def: MacroDefId,
     is_derive: bool,
 ) -> MacroCallId {
-    let mut arg = match macro_attr.input.as_deref() {
-        Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()),
-        _ => Default::default(),
+    let arg = match macro_attr.input.as_deref() {
+        Some(AttrInput::TokenTree(tt, map)) => (
+            {
+                let mut tt = tt.clone();
+                tt.delimiter = tt::Delimiter::UNSPECIFIED;
+                tt
+            },
+            map.clone(),
+        ),
+        _ => (tt::Subtree::empty(), Default::default()),
     };
 
-    // The parentheses are always disposed here.
-    arg.0.delimiter = None;
-
-    let res = def.as_lazy_macro(
+    def.as_lazy_macro(
         db.upcast(),
         krate,
         MacroCallKind::Attr {
@@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
             invoc_attr_index: macro_attr.id,
             is_derive,
         },
-    );
-    res
+    )
 }
 intern::impl_internable!(
     crate::type_ref::TypeRef,

crates/hir-def/src/macro_expansion_tests.rs

@@ -30,7 +30,7 @@ use syntax::{
     SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
     SyntaxNode, TextRange, T,
 };
-use tt::{Subtree, TokenId};
+use tt::token_id::{Subtree, TokenId};
 
 use crate::{
     db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
     tree.token_trees.iter().for_each(|tree| match tree {
         tt::TokenTree::Leaf(leaf) => {
             let id = match leaf {
-                tt::Leaf::Literal(it) => it.id,
-                tt::Leaf::Punct(it) => it.id,
-                tt::Leaf::Ident(it) => it.id,
+                tt::Leaf::Literal(it) => it.span,
+                tt::Leaf::Punct(it) => it.span,
+                tt::Leaf::Ident(it) => it.span,
             };
             ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
         }

crates/hir-def/src/nameres/collector.rs

@@ -46,6 +46,7 @@ use crate::{
     },
     path::{ImportAlias, ModPath, PathKind},
     per_ns::PerNs,
+    tt,
     visibility::{RawVisibility, Visibility},
     AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
     FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
                 .enumerate()
                 .map(|(idx, it)| {
                     // FIXME: a hacky way to create a Name from string.
-                    let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() };
+                    let name =
+                        tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
                     (
                         name.as_name(),
                         ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@@ -451,7 +453,10 @@ impl DefCollector<'_> {
                         directive.module_id,
                         MacroCallKind::Attr {
                             ast_id: ast_id.ast_id,
-                            attr_args: Default::default(),
+                            attr_args: std::sync::Arc::new((
+                                tt::Subtree::empty(),
+                                Default::default(),
+                            )),
                             invoc_attr_index: attr.id,
                             is_derive: false,
                         },
@@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
             let name = match attrs.by_key("rustc_builtin_macro").string_value() {
                 Some(it) => {
                     // FIXME: a hacky way to create a Name from string.
-                    name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name();
+                    name =
+                        tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
                     &name
                 }
                 None => {

crates/hir-def/src/nameres/proc_macro.rs

@@ -1,9 +1,9 @@
 //! Nameres-specific procedural macro data and helpers.
 
 use hir_expand::name::{AsName, Name};
-use tt::{Leaf, TokenTree};
 
 use crate::attr::Attrs;
+use crate::tt::{Leaf, TokenTree};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ProcMacroDef {