Merge #774

774: Batch crate & command r=matklad a=flodiebold

This adds a new crate, `ra_batch`, which is intended for scenarios where you're loading a workspace once and then running some analyses using the HIR API. Also, it adds a command to `ra_cli` which uses that to type-check all crates in a workspace and print some statistics:

E.g. in rust-analyzer:
```
> $ time target/release/ra_cli analysis-stats
Database loaded, 21 roots
Crates in this dir: 28
Total modules found: 231
Total declarations: 3694
Total functions: 2408
Total expressions: 47017
Expressions of unknown type: 19826 (42%)
Expressions of partially unknown type: 4482 (9%)
target/release/ra_cli analysis-stats  3,23s user 0,60s system 100% cpu 3,821 total
```

Or in rust-lang/rust:
```
> $ time ../opensource/rust-analyzer/target/release/ra_cli analysis-stats
Database loaded, 77 roots
Crates in this dir: 130
Total modules found: 1820
Total declarations: 35038
Total functions: 25914
Total expressions: 753678
Expressions of unknown type: 337975 (44%)
Expressions of partially unknown type: 92314 (12%)
../opensource/rust-analyzer/target/release/ra_cli analysis-stats  13,45s user 2,08s system 100% cpu 15,477 total
```

~This still needs a test. Type-checking all of rust-analyzer sadly takes almost a minute when compiled in debug mode 😅 So I'll need to add something simpler (maybe just looking at a few modules).~

Co-authored-by: Florian Diebold <flodiebold@gmail.com>

Author: bors[bot]

Cargo.lock

@@ -0,0 +1,20 @@
+[package]
+edition = "2018"
+name = "ra_batch"
+version = "0.1.0"
+authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
+
+[dependencies]
+log = "0.4.5"
+rustc-hash = "1.0"
+
+failure = "0.1.4"
+
+ra_syntax = { path = "../ra_syntax" }
+ra_db = { path = "../ra_db" }
+ra_hir = { path = "../ra_hir" }
+ra_vfs = { path = "../ra_vfs" }
+ra_project_model = { path = "../ra_project_model" }
+
+[dev-dependencies]
+test_utils = { path = "../test_utils" }

crates/ra_batch/Cargo.toml

@@ -0,0 +1,151 @@
+use std::sync::Arc;
+use std::path::Path;
+use std::collections::HashSet;
+
+use rustc_hash::FxHashMap;
+
+use ra_db::{
+    CrateGraph, FileId, SourceRoot, SourceRootId, SourceDatabase, salsa,
+};
+use ra_hir::{db, HirInterner};
+use ra_project_model::ProjectWorkspace;
+use ra_vfs::{Vfs, VfsChange};
+
+type Result<T> = std::result::Result<T, failure::Error>;
+
+#[salsa::database(
+    ra_db::SourceDatabaseStorage,
+    db::HirDatabaseStorage,
+    db::PersistentHirDatabaseStorage
+)]
+#[derive(Debug)]
+pub struct BatchDatabase {
+    runtime: salsa::Runtime<BatchDatabase>,
+    interner: Arc<HirInterner>,
+}
+
+impl salsa::Database for BatchDatabase {
+    fn salsa_runtime(&self) -> &salsa::Runtime<BatchDatabase> {
+        &self.runtime
+    }
+}
+
+impl AsRef<HirInterner> for BatchDatabase {
+    fn as_ref(&self) -> &HirInterner {
+        &self.interner
+    }
+}
+
+fn vfs_file_to_id(f: ra_vfs::VfsFile) -> FileId {
+    FileId(f.0.into())
+}
+fn vfs_root_to_id(r: ra_vfs::VfsRoot) -> SourceRootId {
+    SourceRootId(r.0.into())
+}
+
+impl BatchDatabase {
+    pub fn load(crate_graph: CrateGraph, vfs: &mut Vfs) -> BatchDatabase {
+        let mut db =
+            BatchDatabase { runtime: salsa::Runtime::default(), interner: Default::default() };
+        db.set_crate_graph(Arc::new(crate_graph));
+
+        // wait until Vfs has loaded all roots
+        let receiver = vfs.task_receiver().clone();
+        let mut roots_loaded = HashSet::new();
+        for task in receiver {
+            vfs.handle_task(task);
+            let mut done = false;
+            for change in vfs.commit_changes() {
+                match change {
+                    VfsChange::AddRoot { root, files } => {
+                        let source_root_id = vfs_root_to_id(root);
+                        log::debug!(
+                            "loaded source root {:?} with path {:?}",
+                            source_root_id,
+                            vfs.root2path(root)
+                        );
+                        let mut file_map = FxHashMap::default();
+                        for (vfs_file, path, text) in files {
+                            let file_id = vfs_file_to_id(vfs_file);
+                            db.set_file_text(file_id, text);
+                            db.set_file_relative_path(file_id, path.clone());
+                            db.set_file_source_root(file_id, source_root_id);
+                            file_map.insert(path, file_id);
+                        }
+                        let source_root = SourceRoot { files: file_map };
+                        db.set_source_root(source_root_id, Arc::new(source_root));
+                        roots_loaded.insert(source_root_id);
+                        if roots_loaded.len() == vfs.num_roots() {
+                            done = true;
+                        }
+                    }
+                    VfsChange::AddFile { .. }
+                    | VfsChange::RemoveFile { .. }
+                    | VfsChange::ChangeFile { .. } => {
+                        // We just need the first scan, so just ignore these
+                    }
+                }
+            }
+            if done {
+                break;
+            }
+        }
+
+        db
+    }
+
+    pub fn load_cargo(root: impl AsRef<Path>) -> Result<(BatchDatabase, Vec<SourceRootId>)> {
+        let root = root.as_ref().canonicalize()?;
+        let ws = ProjectWorkspace::discover(root.as_ref())?;
+        let mut roots = Vec::new();
+        roots.push(root.clone());
+        for pkg in ws.cargo.packages() {
+            roots.push(pkg.root(&ws.cargo).to_path_buf());
+        }
+        for krate in ws.sysroot.crates() {
+            roots.push(krate.root_dir(&ws.sysroot).to_path_buf())
+        }
+        let (mut vfs, roots) = Vfs::new(roots);
+        let mut load = |path: &Path| {
+            let vfs_file = vfs.load(path);
+            log::debug!("vfs file {:?} -> {:?}", path, vfs_file);
+            vfs_file.map(vfs_file_to_id)
+        };
+        let crate_graph = ws.to_crate_graph(&mut load);
+        log::debug!("crate graph: {:?}", crate_graph);
+
+        let local_roots = roots
+            .into_iter()
+            .filter(|r| vfs.root2path(*r).starts_with(&root))
+            .map(vfs_root_to_id)
+            .collect();
+
+        let db = BatchDatabase::load(crate_graph, &mut vfs);
+        let _ = vfs.shutdown();
+        Ok((db, local_roots))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use ra_hir::Crate;
+    use super::*;
+
+    #[test]
+    fn test_loading_rust_analyzer() {
+        let mut path = std::env::current_exe().unwrap();
+        while !path.join("Cargo.toml").is_file() {
+            path = path.parent().unwrap().to_owned();
+        }
+        let (db, roots) = BatchDatabase::load_cargo(path).unwrap();
+        let mut num_crates = 0;
+        for root in roots {
+            for _krate in Crate::source_root_crates(&db, root) {
+                num_crates += 1;
+            }
+        }
+
+        // RA has quite a few crates, but the exact count doesn't matter
+        assert!(num_crates > 20);
+    }
+}

crates/ra_batch/src/lib.rs

@@ -9,6 +9,12 @@ publish = false
 clap = "2.32.0"
 failure = "0.1.4"
 join_to_string = "0.1.1"
+flexi_logger = "0.10.0"
+indicatif = "0.11.0"
+
 ra_syntax = { path = "../ra_syntax" }
 ra_ide_api_light = { path = "../ra_ide_api_light" }
 tools = { path = "../tools" }
+ra_batch = { path = "../ra_batch" }
+ra_hir = { path = "../ra_hir" }
+ra_db = { path = "../ra_db" }

crates/ra_cli/Cargo.toml

@@ -0,0 +1,100 @@
+use std::collections::HashSet;
+
+use ra_db::SourceDatabase;
+use ra_batch::BatchDatabase;
+use ra_hir::{Crate, ModuleDef, Ty, ImplItem};
+use ra_syntax::AstNode;
+
+use crate::Result;
+
+pub fn run(verbose: bool) -> Result<()> {
+    let (db, roots) = BatchDatabase::load_cargo(".")?;
+    println!("Database loaded, {} roots", roots.len());
+    let mut num_crates = 0;
+    let mut visited_modules = HashSet::new();
+    let mut visit_queue = Vec::new();
+    for root in roots {
+        for krate in Crate::source_root_crates(&db, root) {
+            num_crates += 1;
+            let module = krate.root_module(&db).expect("crate in source root without root module");
+            visit_queue.push(module);
+        }
+    }
+    println!("Crates in this dir: {}", num_crates);
+    let mut num_decls = 0;
+    let mut funcs = Vec::new();
+    while let Some(module) = visit_queue.pop() {
+        if visited_modules.insert(module) {
+            visit_queue.extend(module.children(&db));
+
+            for decl in module.declarations(&db) {
+                num_decls += 1;
+                match decl {
+                    ModuleDef::Function(f) => funcs.push(f),
+                    _ => {}
+                }
+            }
+
+            for impl_block in module.impl_blocks(&db) {
+                for item in impl_block.items() {
+                    num_decls += 1;
+                    match item {
+                        ImplItem::Method(f) => funcs.push(*f),
+                        _ => {}
+                    }
+                }
+            }
+        }
+    }
+    println!("Total modules found: {}", visited_modules.len());
+    println!("Total declarations: {}", num_decls);
+    println!("Total functions: {}", funcs.len());
+    let bar = indicatif::ProgressBar::new(funcs.len() as u64);
+    bar.tick();
+    let mut num_exprs = 0;
+    let mut num_exprs_unknown = 0;
+    let mut num_exprs_partially_unknown = 0;
+    for f in funcs {
+        if verbose {
+            let (file_id, source) = f.source(&db);
+            let original_file = file_id.original_file(&db);
+            let path = db.file_relative_path(original_file);
+            let syntax_range = source.syntax().range();
+            let name = f.name(&db);
+            println!("{} ({:?} {})", name, path, syntax_range);
+        }
+        let body = f.body(&db);
+        let inference_result = f.infer(&db);
+        for (expr_id, _) in body.exprs() {
+            let ty = &inference_result[expr_id];
+            num_exprs += 1;
+            if let Ty::Unknown = ty {
+                num_exprs_unknown += 1;
+            } else {
+                let mut is_partially_unknown = false;
+                ty.walk(&mut |ty| {
+                    if let Ty::Unknown = ty {
+                        is_partially_unknown = true;
+                    }
+                });
+                if is_partially_unknown {
+                    num_exprs_partially_unknown += 1;
+                }
+            }
+        }
+        bar.inc(1);
+    }
+    bar.finish_and_clear();
+    println!("Total expressions: {}", num_exprs);
+    println!(
+        "Expressions of unknown type: {} ({}%)",
+        num_exprs_unknown,
+        (num_exprs_unknown * 100 / num_exprs)
+    );
+    println!(
+        "Expressions of partially unknown type: {} ({}%)",
+        num_exprs_partially_unknown,
+        (num_exprs_partially_unknown * 100 / num_exprs)
+    );
+    Ok(())
+}