Merge #3513

3513: Completion in macros r=matklad a=flodiebold

I experimented a bit with completion in macros. It's kind of working, but there are a lot of rough edges.

 - I'm trying to expand the macro call with the inserted fake token. This requires some hacky additions on the HIR level to be able to do "hypothetical" expansions. There should probably be a nicer API for this, if we want to do it this way. I'm not sure whether it's worth it, because we still can't do a lot if the original macro call didn't expand in nearly the same way. E.g. if we have something like `println!("", x<|>)` the expansions will look the same and everything is fine; but in that case we could maybe have achieved the same result in a simpler way. If we have something like `m!(<|>)` where `m!()` doesn't even expand or expands to something very different, we don't really know what to do anyway.
 - Relatedly, there are a lot of cases where this doesn't work because either the original call or the hypothetical call doesn't expand. E.g. if we have `m!(x.<|>)` the original token tree doesn't parse as an expression; if we have `m!(match x { <|> })` the hypothetical token tree doesn't parse. It would be nice if we could have better error recovery in these cases.

Co-authored-by: Florian Diebold <flodiebold@gmail.com>

Author: bors[bot]

Cargo.lock

@@ -12,6 +12,8 @@ log = "0.4.8"
 rustc-hash = "1.1.0"
 either = "1.5.3"
 
+itertools = "0.8.2"
+
 ra_syntax = { path = "../ra_syntax" }
 ra_db = { path = "../ra_db" }
 ra_prof = { path = "../ra_prof" }

crates/ra_hir/Cargo.toml

@@ -6,7 +6,7 @@ use std::{cell::RefCell, fmt, iter::successors};
 
 use hir_def::{
     resolver::{self, HasResolver, Resolver},
-    TraitId,
+    AsMacroCall, TraitId,
 };
 use hir_expand::ExpansionInfo;
 use ra_db::{FileId, FileRange};
@@ -70,6 +70,20 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
         Some(node)
     }
 
+    pub fn expand_hypothetical(
+        &self,
+        actual_macro_call: &ast::MacroCall,
+        hypothetical_args: &ast::TokenTree,
+        token_to_map: SyntaxToken,
+    ) -> Option<(SyntaxNode, SyntaxToken)> {
+        let macro_call =
+            self.find_file(actual_macro_call.syntax().clone()).with_value(actual_macro_call);
+        let sa = self.analyze2(macro_call.map(|it| it.syntax()), None);
+        let macro_call_id = macro_call
+            .as_call_id(self.db, |path| sa.resolver.resolve_path_as_macro(self.db, &path))?;
+        hir_expand::db::expand_hypothetical(self.db, macro_call_id, hypothetical_args, token_to_map)
+    }
+
     pub fn descend_into_macros(&self, token: SyntaxToken) -> SyntaxToken {
         let parent = token.parent();
         let parent = self.find_file(parent);
@@ -104,6 +118,25 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
         node.ancestors_with_macros(self.db).map(|it| it.value)
     }
 
+    pub fn ancestors_at_offset_with_macros(
+        &self,
+        node: &SyntaxNode,
+        offset: TextUnit,
+    ) -> impl Iterator<Item = SyntaxNode> + '_ {
+        use itertools::Itertools;
+        node.token_at_offset(offset)
+            .map(|token| self.ancestors_with_macros(token.parent()))
+            .kmerge_by(|node1, node2| node1.text_range().len() < node2.text_range().len())
+    }
+
+    pub fn find_node_at_offset_with_macros<N: AstNode>(
+        &self,
+        node: &SyntaxNode,
+        offset: TextUnit,
+    ) -> Option<N> {
+        self.ancestors_at_offset_with_macros(node, offset).find_map(N::cast)
+    }
+
     pub fn type_of_expr(&self, expr: &ast::Expr) -> Option<Type> {
         self.analyze(expr.syntax()).type_of(self.db, &expr)
     }

crates/ra_hir/src/semantics.rs

@@ -72,6 +72,30 @@ pub trait AstDatabase: SourceDatabase {
     fn intern_eager_expansion(&self, eager: EagerCallLoc) -> EagerMacroId;
 }
 
+/// This expands the given macro call, but with different arguments. This is
+/// used for completion, where we want to see what 'would happen' if we insert a
+/// token. The `token_to_map` mapped down into the expansion, with the mapped
+/// token returned.
+pub fn expand_hypothetical(
+    db: &impl AstDatabase,
+    actual_macro_call: MacroCallId,
+    hypothetical_args: &ra_syntax::ast::TokenTree,
+    token_to_map: ra_syntax::SyntaxToken,
+) -> Option<(SyntaxNode, ra_syntax::SyntaxToken)> {
+    let macro_file = MacroFile { macro_call_id: actual_macro_call };
+    let (tt, tmap_1) = mbe::syntax_node_to_token_tree(hypothetical_args.syntax()).unwrap();
+    let range =
+        token_to_map.text_range().checked_sub(hypothetical_args.syntax().text_range().start())?;
+    let token_id = tmap_1.token_by_range(range)?;
+    let macro_def = expander(db, actual_macro_call)?;
+    let (node, tmap_2) =
+        parse_macro_with_arg(db, macro_file, Some(std::sync::Arc::new((tt, tmap_1))))?;
+    let token_id = macro_def.0.map_id_down(token_id);
+    let range = tmap_2.range_by_token(token_id)?.by_kind(token_to_map.kind())?;
+    let token = ra_syntax::algo::find_covering_element(&node.syntax_node(), range).into_token()?;
+    Some((node.syntax_node(), token))
+}
+
 pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
     let map =
         db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it));
@@ -129,16 +153,43 @@ pub(crate) fn macro_arg(
 pub(crate) fn macro_expand(
     db: &dyn AstDatabase,
     id: MacroCallId,
+) -> Result<Arc<tt::Subtree>, String> {
+    macro_expand_with_arg(db, id, None)
+}
+
+fn expander(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
+    let lazy_id = match id {
+        MacroCallId::LazyMacro(id) => id,
+        MacroCallId::EagerMacro(_id) => {
+            return None;
+        }
+    };
+
+    let loc = db.lookup_intern_macro(lazy_id);
+    let macro_rules = db.macro_def(loc.def)?;
+    Some(macro_rules)
+}
+
+fn macro_expand_with_arg(
+    db: &dyn AstDatabase,
+    id: MacroCallId,
+    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
 ) -> Result<Arc<tt::Subtree>, String> {
     let lazy_id = match id {
         MacroCallId::LazyMacro(id) => id,
         MacroCallId::EagerMacro(id) => {
-            return Ok(db.lookup_intern_eager_expansion(id).subtree);
+            if arg.is_some() {
+                return Err(
+                    "hypothetical macro expansion not implemented for eager macro".to_owned()
+                );
+            } else {
+                return Ok(db.lookup_intern_eager_expansion(id).subtree);
+            }
         }
     };
 
     let loc = db.lookup_intern_macro(lazy_id);
-    let macro_arg = db.macro_arg(id).ok_or("Fail to args in to tt::TokenTree")?;
+    let macro_arg = arg.or_else(|| db.macro_arg(id)).ok_or("Fail to args in to tt::TokenTree")?;
 
     let macro_rules = db.macro_def(loc.def).ok_or("Fail to find macro definition")?;
     let tt = macro_rules.0.expand(db, lazy_id, &macro_arg.0).map_err(|err| format!("{:?}", err))?;
@@ -162,12 +213,24 @@ pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Optio
 pub(crate) fn parse_macro(
     db: &dyn AstDatabase,
     macro_file: MacroFile,
+) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
+    parse_macro_with_arg(db, macro_file, None)
+}
+
+pub fn parse_macro_with_arg(
+    db: &dyn AstDatabase,
+    macro_file: MacroFile,
+    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
 ) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
     let _p = profile("parse_macro_query");
 
     let macro_call_id = macro_file.macro_call_id;
-    let tt = db
-        .macro_expand(macro_call_id)
+    let expansion = if let Some(arg) = arg {
+        macro_expand_with_arg(db, macro_call_id, Some(arg))
+    } else {
+        db.macro_expand(macro_call_id)
+    };
+    let tt = expansion
         .map_err(|err| {
             // Note:
             // The final goal we would like to make all parse_macro success,

crates/ra_hir_expand/src/db.rs

@@ -38,7 +38,7 @@ pub(super) fn complete_dot(acc: &mut Completions, ctx: &CompletionContext) {
 fn complete_fields(acc: &mut Completions, ctx: &CompletionContext, receiver: &Type) {
     for receiver in receiver.autoderef(ctx.db) {
         for (field, ty) in receiver.fields(ctx.db) {
-            if ctx.module.map_or(false, |m| !field.is_visible_from(ctx.db, m)) {
+            if ctx.scope().module().map_or(false, |m| !field.is_visible_from(ctx.db, m)) {
                 // Skip private field. FIXME: If the definition location of the
                 // field is editable, we should show the completion
                 continue;
@@ -53,7 +53,7 @@ fn complete_fields(acc: &mut Completions, ctx: &CompletionContext, receiver: &Ty
 }
 
 fn complete_methods(acc: &mut Completions, ctx: &CompletionContext, receiver: &Type) {
-    if let Some(krate) = ctx.module.map(|it| it.krate()) {
+    if let Some(krate) = ctx.krate {
         let mut seen_methods = FxHashSet::default();
         let traits_in_scope = ctx.scope().traits_in_scope();
         receiver.iterate_method_candidates(ctx.db, krate, &traits_in_scope, None, |_ty, func| {
@@ -620,4 +620,102 @@ mod tests {
         "###
         );
     }
+
+    #[test]
+    fn works_in_simple_macro_1() {
+        assert_debug_snapshot!(
+            do_ref_completion(
+                r"
+                macro_rules! m { ($e:expr) => { $e } }
+                struct A { the_field: u32 }
+                fn foo(a: A) {
+                    m!(a.x<|>)
+                }
+                ",
+            ),
+            @r###"
+        [
+            CompletionItem {
+                label: "the_field",
+                source_range: [156; 157),
+                delete: [156; 157),
+                insert: "the_field",
+                kind: Field,
+                detail: "u32",
+            },
+        ]
+        "###
+        );
+    }
+
+    #[test]
+    fn works_in_simple_macro_recursive() {
+        assert_debug_snapshot!(
+            do_ref_completion(
+                r"
+                macro_rules! m { ($e:expr) => { $e } }
+                struct A { the_field: u32 }
+                fn foo(a: A) {
+                    m!(a.x<|>)
+                }
+                ",
+            ),
+            @r###"
+        [
+            CompletionItem {
+                label: "the_field",
+                source_range: [156; 157),
+                delete: [156; 157),
+                insert: "the_field",
+                kind: Field,
+                detail: "u32",
+            },
+        ]
+        "###
+        );
+    }
+
+    #[test]
+    fn works_in_simple_macro_2() {
+        // this doesn't work yet because the macro doesn't expand without the token -- maybe it can be fixed with better recovery
+        assert_debug_snapshot!(
+            do_ref_completion(
+                r"
+                macro_rules! m { ($e:expr) => { $e } }
+                struct A { the_field: u32 }
+                fn foo(a: A) {
+                    m!(a.<|>)
+                }
+                ",
+            ),
+            @r###"[]"###
+        );
+    }
+
+    #[test]
+    fn works_in_simple_macro_recursive_1() {
+        assert_debug_snapshot!(
+            do_ref_completion(
+                r"
+                macro_rules! m { ($e:expr) => { $e } }
+                struct A { the_field: u32 }
+                fn foo(a: A) {
+                    m!(m!(m!(a.x<|>)))
+                }
+                ",
+            ),
+            @r###"
+        [
+            CompletionItem {
+                label: "the_field",
+                source_range: [162; 163),
+                delete: [162; 163),
+                insert: "the_field",
+                kind: Field,
+                detail: "u32",
+            },
+        ]
+        "###
+        );
+    }
 }

crates/ra_ide/src/completion/complete_dot.rs

@@ -79,6 +79,7 @@ pub(super) fn complete_expr_keyword(acc: &mut Completions, ctx: &CompletionConte
 }
 
 fn is_in_loop_body(leaf: &SyntaxToken) -> bool {
+    // FIXME move this to CompletionContext and make it handle macros
     for node in leaf.parent().ancestors() {
         if node.kind() == FN_DEF || node.kind() == LAMBDA_EXPR {
             break;

crates/ra_ide/src/completion/complete_keyword.rs

@@ -1,4 +1,4 @@
-//! Completion of paths, including when writing a single name.
+//! Completion of paths, i.e. `some::prefix::<|>`.
 
 use hir::{Adt, HasVisibility, PathResolution, ScopeDef};
 use ra_syntax::AstNode;
@@ -48,7 +48,7 @@ pub(super) fn complete_path(acc: &mut Completions, ctx: &CompletionContext) {
             };
             // Iterate assoc types separately
             // FIXME: complete T::AssocType
-            let krate = ctx.module.map(|m| m.krate());
+            let krate = ctx.krate;
             if let Some(krate) = krate {
                 let traits_in_scope = ctx.scope().traits_in_scope();
                 ty.iterate_path_candidates(ctx.db, krate, &traits_in_scope, None, |_ty, item| {
@@ -934,4 +934,37 @@ mod tests {
         "###
         );
     }
+
+    #[test]
+    fn completes_in_simple_macro_call() {
+        let completions = do_reference_completion(
+            r#"
+                macro_rules! m { ($e:expr) => { $e } }
+                fn main() { m!(self::f<|>); }
+                fn foo() {}
+            "#,
+        );
+        assert_debug_snapshot!(completions, @r###"
+        [
+            CompletionItem {
+                label: "foo()",
+                source_range: [93; 94),
+                delete: [93; 94),
+                insert: "foo()$0",
+                kind: Function,
+                lookup: "foo",
+                detail: "fn foo()",
+            },
+            CompletionItem {
+                label: "main()",
+                source_range: [93; 94),
+                delete: [93; 94),
+                insert: "main()$0",
+                kind: Function,
+                lookup: "main",
+                detail: "fn main()",
+            },
+        ]
+        "###);
+    }
 }

crates/ra_ide/src/completion/complete_path.rs

@@ -86,4 +86,22 @@ mod tests {
         ]
         "###);
     }
+
+    #[test]
+    fn completes_in_simple_macro_call() {
+        // FIXME: doesn't work yet because of missing error recovery in macro expansion
+        let completions = complete(
+            r"
+            macro_rules! m { ($e:expr) => { $e } }
+            enum E { X }
+
+            fn foo() {
+               m!(match E::X {
+                   <|>
+               })
+            }
+            ",
+        );
+        assert_debug_snapshot!(completions, @r###"[]"###);
+    }
 }