Change scraping strategy to embed in existing unit graph, add CompileMode::Docscrape

Author: willcrichton

src/cargo/core/compiler/build_config.rs

@@ -149,6 +149,7 @@ pub enum CompileMode {
     Doc { deps: bool },
     /// A target that will be tested with `rustdoc`.
     Doctest,
+    Docscrape,
     /// A marker for Units that represent the execution of a `build.rs` script.
     RunCustomBuild,
 }
@@ -166,6 +167,7 @@ impl ser::Serialize for CompileMode {
             Bench => "bench".serialize(s),
             Doc { .. } => "doc".serialize(s),
             Doctest => "doctest".serialize(s),
+            Docscrape => "docscrape".serialize(s),
             RunCustomBuild => "run-custom-build".serialize(s),
         }
     }
@@ -187,6 +189,11 @@ impl CompileMode {
         self == CompileMode::Doctest
     }
 
+    /// Returns `true` if this is scraping examples for documentation.
+    pub fn is_doc_scrape(self) -> bool {
+        self == CompileMode::Docscrape
+    }
+
     /// Returns `true` if this is any type of test (test, benchmark, doc test, or
     /// check test).
     pub fn is_any_test(self) -> bool {

src/cargo/core/compiler/build_context/mod.rs

@@ -48,6 +48,8 @@ pub struct BuildContext<'a, 'cfg> {
     /// The dependency graph of units to compile.
     pub unit_graph: UnitGraph,
 
+    pub scrape_units: Vec<Unit>,
+
     /// The list of all kinds that are involved in this build
     pub all_kinds: HashSet<CompileKind>,
 }
@@ -62,6 +64,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
         target_data: RustcTargetData<'cfg>,
         roots: Vec<Unit>,
         unit_graph: UnitGraph,
+        scrape_units: Vec<Unit>,
     ) -> CargoResult<BuildContext<'a, 'cfg>> {
         let all_kinds = unit_graph
             .keys()
@@ -80,6 +83,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
             target_data,
             roots,
             unit_graph,
+            scrape_units,
             all_kinds,
         })
     }

src/cargo/core/compiler/build_context/target_info.rs

@@ -461,7 +461,10 @@ impl TargetInfo {
                 }
             }
             CompileMode::Check { .. } => Ok((vec![FileType::new_rmeta()], Vec::new())),
-            CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
+            CompileMode::Doc { .. }
+            | CompileMode::Doctest
+            | CompileMode::Docscrape
+            | CompileMode::RunCustomBuild => {
                 panic!("asked for rustc output for non-rustc mode")
             }
         }

src/cargo/core/compiler/context/compilation_files.rs

@@ -417,6 +417,10 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
                 // but Cargo does not know about that.
                 vec![]
             }
+            CompileMode::Docscrape => {
+                // FIXME(wcrichto): should this include the *.examples files?
+                vec![]
+            }
             CompileMode::Test
             | CompileMode::Build
             | CompileMode::Bench

src/cargo/core/compiler/job_queue.rs

@@ -1001,6 +1001,7 @@ impl<'cfg> DrainState<'cfg> {
         let target_name = unit.target.name();
         match unit.mode {
             CompileMode::Doc { .. } => format!("{}(doc)", pkg_name),
+            CompileMode::Docscrape { .. } => format!("{}(docscrape)", pkg_name),
             CompileMode::RunCustomBuild => format!("{}(build)", pkg_name),
             CompileMode::Test | CompileMode::Check { test: true } => match unit.target.kind() {
                 TargetKind::Lib(_) => format!("{}(test)", target_name),

src/cargo/core/compiler/mod.rs

@@ -165,7 +165,7 @@ fn compile<'cfg>(
         let force = exec.force_rebuild(unit) || force_rebuild;
         let mut job = fingerprint::prepare_target(cx, unit, force)?;
         job.before(if job.freshness() == Freshness::Dirty {
-            let work = if unit.mode.is_doc() {
+            let work = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
                 rustdoc(cx, unit)?
             } else {
                 rustc(cx, unit, exec)?
@@ -647,6 +647,48 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
         rustdoc.args(args);
     }
 
+    let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
+        let layout = cx.files().layout(unit.kind);
+        let output_dir = layout.prepare_tmp()?;
+        Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
+    };
+
+    if unit.mode.is_doc_scrape() {
+        rustdoc.arg("-Zunstable-options");
+
+        rustdoc
+            .arg("--scrape-examples-output-path")
+            .arg(scrape_output_path(unit)?);
+
+        for root in &cx.bcx.roots {
+            rustdoc
+                .arg("--scrape-examples-target-crate")
+                .arg(root.pkg.name());
+        }
+    } else if cx.bcx.scrape_units.len() > 0 {
+        rustdoc.arg("-Zunstable-options");
+
+        for scrape_unit in &cx.bcx.scrape_units {
+            rustdoc
+                .arg("--with-examples")
+                .arg(scrape_output_path(scrape_unit)?);
+        }
+
+        let mut all_units = cx
+            .bcx
+            .unit_graph
+            .values()
+            .map(|deps| deps.iter().map(|dep| &dep.unit))
+            .flatten();
+        let check_unit = all_units
+            .find(|other| {
+                unit.pkg == other.pkg && unit.target == other.target && other.mode.is_check()
+            })
+            .with_context(|| format!("Could not find check unit for {:?}", unit))?;
+        let metadata = cx.files().metadata(check_unit);
+        rustdoc.arg("-C").arg(format!("metadata={}", metadata));
+    }
+
     build_deps_args(&mut rustdoc, cx, unit)?;
     rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;
 

src/cargo/core/compiler/timings.rs

@@ -176,6 +176,7 @@ impl<'cfg> Timings<'cfg> {
             CompileMode::Bench => target.push_str(" (bench)"),
             CompileMode::Doc { .. } => target.push_str(" (doc)"),
             CompileMode::Doctest => target.push_str(" (doc test)"),
+            CompileMode::Docscrape => target.push_str(" (doc scrape)"),
             CompileMode::RunCustomBuild => target.push_str(" (run)"),
         }
         let unit_time = UnitTime {

src/cargo/core/compiler/unit_dependencies.rs

@@ -47,6 +47,7 @@ struct State<'a, 'cfg> {
     target_data: &'a RustcTargetData<'cfg>,
     profiles: &'a Profiles,
     interner: &'a UnitInterner,
+    scrape_roots: &'a [Unit],
 
     /// A set of edges in `unit_dependencies` where (a, b) means that the
     /// dependency from a to b was added purely because it was a dev-dependency.
@@ -61,6 +62,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
     features: &'a ResolvedFeatures,
     std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
     roots: &[Unit],
+    scrape_roots: &[Unit],
     std_roots: &HashMap<CompileKind, Vec<Unit>>,
     global_mode: CompileMode,
     target_data: &'a RustcTargetData<'cfg>,
@@ -91,12 +93,14 @@ pub fn build_unit_dependencies<'a, 'cfg>(
         target_data,
         profiles,
         interner,
+        scrape_roots,
         dev_dependency_edges: HashSet::new(),
     };
 
     let std_unit_deps = calc_deps_of_std(&mut state, std_roots)?;
 
     deps_of_roots(roots, &mut state)?;
+    deps_of_roots(scrape_roots, &mut state)?;
     super::links::validate_links(state.resolve(), &state.unit_dependencies)?;
     // Hopefully there aren't any links conflicts with the standard library?
 
@@ -477,6 +481,17 @@ fn compute_deps_doc(
     if unit.target.is_bin() || unit.target.is_example() {
         ret.extend(maybe_lib(unit, state, unit_for)?);
     }
+
+    for scrape_unit in state.scrape_roots.iter() {
+        ret.push(UnitDep {
+            unit: scrape_unit.clone(),
+            unit_for: unit_for.with_dependency(scrape_unit, &scrape_unit.target),
+            extern_crate_name: InternedString::new(""),
+            public: false,
+            noprelude: false,
+        });
+    }
+
     Ok(ret)
 }
 
@@ -568,7 +583,7 @@ fn dep_build_script(
 /// Choose the correct mode for dependencies.
 fn check_or_build_mode(mode: CompileMode, target: &Target) -> CompileMode {
     match mode {
-        CompileMode::Check { .. } | CompileMode::Doc { .. } => {
+        CompileMode::Check { .. } | CompileMode::Doc { .. } | CompileMode::Docscrape => {
             if target.for_host() {
                 // Plugin and proc macro targets should be compiled like
                 // normal.

src/cargo/core/profiles.rs

@@ -323,7 +323,7 @@ impl Profiles {
                         (InternedString::new("dev"), None)
                     }
                 }
-                CompileMode::Doc { .. } => (InternedString::new("doc"), None),
+                CompileMode::Doc { .. } | CompileMode::Docscrape => (InternedString::new("doc"), None),
             }
         } else {
             (self.requested_profile, None)