assign ids to tokens

Author: matklad

crates/ra_mbe/src/mbe_expander.rs

@@ -3,6 +3,7 @@
 /// `tt::TokenTree` for the result of the expansion.
 use rustc_hash::FxHashMap;
 use ra_syntax::SmolStr;
+use tt::TokenId;
 
 use crate::tt_cursor::TtCursor;
 
@@ -185,7 +186,8 @@ fn expand_tt(
         }
         crate::TokenTree::Leaf(leaf) => match leaf {
             crate::Leaf::Ident(ident) => {
-                tt::Leaf::from(tt::Ident { text: ident.text.clone() }).into()
+                tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
+                    .into()
             }
             crate::Leaf::Punct(punct) => tt::Leaf::from(punct.clone()).into(),
             crate::Leaf::Var(v) => bindings.get(&v.text, nesting)?.clone(),

crates/ra_mbe/src/mbe_parser.rs

@@ -41,7 +41,7 @@ fn parse_subtree(tt: &tt::Subtree) -> Option<crate::Subtree> {
                     }
                 }
                 tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(),
-                tt::Leaf::Ident(tt::Ident { text }) => {
+                tt::Leaf::Ident(tt::Ident { text, id: _ }) => {
                     crate::Leaf::from(crate::Ident { text: text.clone() }).into()
                 }
                 tt::Leaf::Literal(tt::Literal { text }) => {

crates/ra_mbe/src/syntax_bridge.rs

@@ -37,7 +37,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
                 convert_tt(child)?.into()
             } else if child.kind().is_keyword() || child.kind() == IDENT {
                 let text = child.leaf_text().unwrap().clone();
-                tt::Leaf::from(tt::Ident { text }).into()
+                tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into()
             } else if child.kind().is_literal() {
                 tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into()
             } else {

crates/ra_tt/src/lib.rs

@@ -18,6 +18,15 @@ use std::fmt;
 
 use smol_str::SmolStr;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TokenId(pub u32);
+
+impl TokenId {
+    pub const fn unspecified() -> TokenId {
+        TokenId(!0)
+    }
+}
+
 #[derive(Debug, Clone)]
 pub enum TokenTree {
     Leaf(Leaf),
@@ -67,6 +76,7 @@ pub enum Spacing {
 #[derive(Debug, Clone)]
 pub struct Ident {
     pub text: SmolStr,
+    pub id: TokenId,
 }
 
 impl fmt::Display for TokenTree {