assign ids when converting tt

Author: matklad

crates/ra_mbe/src/lib.rs

@@ -144,8 +144,8 @@ impl_froms!(TokenTree: Leaf, Subtree);
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
-        let definition_tt = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
-        let invocation_tt = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
+        let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
+        let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
         let rules = crate::MacroRules::parse(&definition_tt).unwrap();
         let expansion = rules.expand(&invocation_tt).unwrap();
         assert_eq!(
@@ -160,7 +160,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         let macro_definition =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
-        let definition_tt = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
+        let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
         crate::MacroRules::parse(&definition_tt).unwrap()
     }
 
@@ -169,7 +169,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
-        let invocation_tt = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
+        let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
 
         let expaned = rules.expand(&invocation_tt).unwrap();
         assert_eq!(expaned.to_string(), expansion);

crates/ra_mbe/src/syntax_bridge.rs

@@ -1,10 +1,34 @@
-use ra_syntax::{ast, AstNode, SyntaxNode, SyntaxKind::*};
+use ra_syntax::{
+    AstNode, SyntaxNode, TextRange,
+    ast, SyntaxKind::*, TextUnit
+};
 
-pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<tt::Subtree> {
-    convert_tt(ast.syntax())
+#[derive(Default)]
+pub struct TokenMap {
+    /// Maps `tt::TokenId` to the *relative* source range.
+    toknes: Vec<TextRange>,
 }
 
-fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
+pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> {
+    let mut token_map = TokenMap::default();
+    let node = ast.syntax();
+    let tt = convert_tt(&mut token_map, node.range().start(), node)?;
+    Some((tt, token_map))
+}
+
+impl TokenMap {
+    fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId {
+        let id = self.toknes.len();
+        self.toknes.push(relative_range);
+        tt::TokenId(id as u32)
+    }
+}
+
+fn convert_tt(
+    token_map: &mut TokenMap,
+    global_offset: TextUnit,
+    tt: &SyntaxNode,
+) -> Option<tt::Subtree> {
     let first_child = tt.first_child()?;
     let last_child = tt.last_child()?;
     let delimiter = match (first_child.kind(), last_child.kind()) {
@@ -34,10 +58,12 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
             }
         } else {
             let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
-                convert_tt(child)?.into()
+                convert_tt(token_map, global_offset, child)?.into()
             } else if child.kind().is_keyword() || child.kind() == IDENT {
+                let relative_range = child.range() - global_offset;
+                let id = token_map.alloc(relative_range);
                 let text = child.leaf_text().unwrap().clone();
-                tt::Leaf::from(tt::Ident { text, id: tt::TokenId::unspecified() }).into()
+                tt::Leaf::from(tt::Ident { text, id }).into()
             } else if child.kind().is_literal() {
                 tt::Leaf::from(tt::Literal { text: child.leaf_text().unwrap().clone() }).into()
             } else {