Change rustdoc-scrape-examples to be a target-level configuration

Author: willcrichton

src/cargo/core/compiler/job_queue.rs

@@ -431,8 +431,9 @@ impl<'cfg> JobQueue<'cfg> {
             .filter(|dep| {
                 // Binaries aren't actually needed to *compile* tests, just to run
                 // them, so we don't include this dependency edge in the job graph.
+                // But we shouldn't filter out dependencies being scraped for Rustdoc.
                 (!dep.unit.target.is_test() && !dep.unit.target.is_bin())
-                    || dep.unit.artifact.is_true()
+                    || dep.unit.artifact.is_true() || dep.unit.mode.is_doc_scrape()
             })
             .map(|dep| {
                 // Handle the case here where our `unit -> dep` dependency may

src/cargo/core/compiler/mod.rs

@@ -666,6 +666,10 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
     if unit.mode.is_doc_scrape() {
         debug_assert!(cx.bcx.scrape_units.contains(unit));
 
+        if unit.target.is_test() {
+            rustdoc.arg("--test");
+        }
+
         rustdoc.arg("-Zunstable-options");
 
         rustdoc

src/cargo/core/features.rs

@@ -661,9 +661,6 @@ unstable_cli_options!(
     separate_nightlies: bool = (HIDDEN),
     terminal_width: Option<Option<usize>>  = ("Provide a terminal width to rustc for error truncation"),
     unstable_options: bool = ("Allow the usage of unstable options"),
-    // TODO(wcrichto): move scrape example configuration into Cargo.toml before stabilization
-    // See: https://github.com/rust-lang/cargo/pull/9525#discussion_r728470927
-    rustdoc_scrape_examples: Option<String> = ("Allow rustdoc to scrape examples from reverse-dependencies for documentation"),
     skip_rustdoc_fingerprint: bool = (HIDDEN),
 );
 
@@ -887,15 +884,6 @@ impl CliUnstable {
             "namespaced-features" => stabilized_warn(k, "1.60", STABILISED_NAMESPACED_FEATURES),
             "weak-dep-features" => stabilized_warn(k, "1.60", STABILIZED_WEAK_DEP_FEATURES),
             "credential-process" => self.credential_process = parse_empty(k, v)?,
-            "rustdoc-scrape-examples" => {
-                if let Some(s) = v {
-                    self.rustdoc_scrape_examples = Some(s.to_string())
-                } else {
-                    bail!(
-                        r#"-Z rustdoc-scrape-examples must take "all" or "examples" as an argument"#
-                    )
-                }
-            }
             "skip-rustdoc-fingerprint" => self.skip_rustdoc_fingerprint = parse_empty(k, v)?,
             "compile-progress" => stabilized_warn(k, "1.30", STABILIZED_COMPILE_PROGRESS),
             "offline" => stabilized_err(k, "1.36", STABILIZED_OFFLINE)?,

src/cargo/core/manifest.rs

@@ -220,6 +220,7 @@ struct TargetInner {
     for_host: bool,
     proc_macro: bool,
     edition: Edition,
+    doc_scrape_examples: Option<bool>,
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)]
@@ -373,6 +374,7 @@ compact_debug! {
                 for_host
                 proc_macro
                 edition
+                doc_scrape_examples
             )]
         }
     }
@@ -648,6 +650,7 @@ impl Target {
                 harness: true,
                 for_host: false,
                 proc_macro: false,
+                doc_scrape_examples: None,
                 edition,
                 tested: true,
                 benched: true,
@@ -804,6 +807,14 @@ impl Target {
     pub fn edition(&self) -> Edition {
         self.inner.edition
     }
+    pub fn doc_scrape_examples(&self) -> bool {
+        self.inner.doc_scrape_examples.unwrap_or_else(|| {
+            matches!(
+                self.kind(),
+                TargetKind::Lib(..) | TargetKind::ExampleBin | TargetKind::ExampleLib(..)
+            )
+        })
+    }
     pub fn benched(&self) -> bool {
         self.inner.benched
     }
@@ -918,6 +929,10 @@ impl Target {
         Arc::make_mut(&mut self.inner).edition = edition;
         self
     }
+    pub fn set_doc_scrape_examples(&mut self, doc_scrape_examples: bool) -> &mut Target {
+        Arc::make_mut(&mut self.inner).doc_scrape_examples = Some(doc_scrape_examples);
+        self
+    }
     pub fn set_harness(&mut self, harness: bool) -> &mut Target {
         Arc::make_mut(&mut self.inner).harness = harness;
         self

src/cargo/ops/cargo_compile.rs

@@ -45,7 +45,7 @@ use crate::util::interning::InternedString;
 use crate::util::restricted_names::is_glob_pattern;
 use crate::util::{closest_msg, profile, CargoResult, StableHasher};
 
-use anyhow::{bail, Context as _};
+use anyhow::Context as _;
 
 /// Contains information about how a package should be compiled.
 ///
@@ -370,8 +370,7 @@ pub fn create_bcx<'a, 'cfg>(
     let target_data = RustcTargetData::new(ws, &build_config.requested_kinds)?;
 
     let all_packages = &Packages::All;
-    let rustdoc_scrape_examples = &config.cli_unstable().rustdoc_scrape_examples;
-    let need_reverse_dependencies = rustdoc_scrape_examples.is_some();
+    let need_reverse_dependencies = matches!(build_config.mode, CompileMode::Doc { .. });
     let full_specs = if need_reverse_dependencies {
         all_packages
     } else {
@@ -511,47 +510,26 @@ pub fn create_bcx<'a, 'cfg>(
         override_rustc_crate_types(&mut units, args, interner)?;
     }
 
-    let mut scrape_units = match rustdoc_scrape_examples {
-        Some(arg) => {
-            let filter = match arg.as_str() {
-                "all" => CompileFilter::new_all_targets(),
-                "examples" => CompileFilter::new(
-                    LibRule::False,
-                    FilterRule::none(),
-                    FilterRule::none(),
-                    FilterRule::All,
-                    FilterRule::none(),
-                ),
-                _ => {
-                    bail!(
-                        r#"-Z rustdoc-scrape-examples must take "all" or "examples" as an argument"#
-                    )
-                }
-            };
-            let to_build_ids = resolve.specs_to_ids(&resolve_specs)?;
-            let to_builds = pkg_set.get_many(to_build_ids)?;
-            let mode = CompileMode::Docscrape;
-
-            generate_targets(
-                ws,
-                &to_builds,
-                &filter,
-                &build_config.requested_kinds,
-                explicit_host_kind,
-                mode,
-                &resolve,
-                &workspace_resolve,
-                &resolved_features,
-                &pkg_set,
-                &profiles,
-                interner,
-            )?
-            .into_iter()
-            // Proc macros should not be scraped for functions, since they only export macros
-            .filter(|unit| !unit.target.proc_macro())
-            .collect::<Vec<_>>()
-        }
-        None => Vec::new(),
+    let mut scrape_units = {
+        let to_build_ids = resolve.specs_to_ids(&resolve_specs)?;
+        let to_builds = pkg_set.get_many(to_build_ids)?;
+
+        generate_targets(
+            ws,
+            &to_builds,
+            &CompileFilter::Default {
+                required_features_filterable: false,
+            },
+            &build_config.requested_kinds,
+            explicit_host_kind,
+            CompileMode::Docscrape,
+            &resolve,
+            &workspace_resolve,
+            &resolved_features,
+            &pkg_set,
+            &profiles,
+            interner,
+        )?
     };
 
     let std_roots = if let Some(crates) = &config.cli_unstable().build_std {
@@ -1468,7 +1446,8 @@ fn filter_default_targets(targets: &[Target], mode: CompileMode) -> Vec<&Target>
                 })
                 .collect()
         }
-        CompileMode::Doctest | CompileMode::Docscrape | CompileMode::RunCustomBuild => {
+        CompileMode::Docscrape => targets.iter().filter(|t| t.doc_scrape_examples()).collect(),
+        CompileMode::Doctest | CompileMode::RunCustomBuild => {
             panic!("Invalid mode {:?}", mode)
         }
     }

src/cargo/util/toml/mod.rs

@@ -2682,6 +2682,8 @@ struct TomlTarget {
     bench: Option<bool>,
     doc: Option<bool>,
     plugin: Option<bool>,
+    #[serde(rename = "doc-scrape-examples")]
+    doc_scrape_examples: Option<bool>,
     #[serde(rename = "proc-macro")]
     proc_macro_raw: Option<bool>,
     #[serde(rename = "proc_macro")]

src/cargo/util/toml/targets.rs

@@ -808,6 +808,10 @@ fn configure(toml: &TomlTarget, target: &mut Target) -> CargoResult<()> {
         .set_benched(toml.bench.unwrap_or_else(|| t2.benched()))
         .set_harness(toml.harness.unwrap_or_else(|| t2.harness()))
         .set_proc_macro(toml.proc_macro().unwrap_or_else(|| t2.proc_macro()))
+        .set_doc_scrape_examples(
+            toml.doc_scrape_examples
+                .unwrap_or_else(|| t2.doc_scrape_examples()),
+        )
         .set_for_host(match (toml.plugin, toml.proc_macro()) {
             (None, None) => t2.for_host(),
             (Some(true), _) | (_, Some(true)) => true,

tests/testsuite/doc.rs

@@ -2378,7 +2378,7 @@ fn scrape_examples_basic() {
         .file("src/lib.rs", "pub fn foo() {}\npub fn bar() { foo(); }")
         .build();
 
-    p.cargo("doc -Zunstable-options -Z rustdoc-scrape-examples=all")
+    p.cargo("doc")
         .masquerade_as_nightly_cargo()
         .with_stderr(
             "\
@@ -2439,9 +2439,7 @@ fn scrape_examples_avoid_build_script_cycle() {
         .file("bar/build.rs", "fn main(){}")
         .build();
 
-    p.cargo("doc --all -Zunstable-options -Z rustdoc-scrape-examples=all")
-        .masquerade_as_nightly_cargo()
-        .run();
+    p.cargo("doc --all").masquerade_as_nightly_cargo().run();
 }
 
 #[cargo_test]
@@ -2501,9 +2499,34 @@ fn scrape_examples_complex_reverse_dependencies() {
         .file("b/src/lib.rs", "")
         .build();
 
-    p.cargo("doc -Zunstable-options -Z rustdoc-scrape-examples=all")
-        .masquerade_as_nightly_cargo()
-        .run();
+    p.cargo("doc").masquerade_as_nightly_cargo().run();
+}
+
+#[cargo_test]
+fn scrape_examples_conflicting_reverse_deps() {
+    if !is_nightly() {
+        // -Z rustdoc-scrape-examples is unstable
+        return;
+    }
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                authors = []
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file("examples/a.rs", "fn main() {}")
+        .file("benches/a.rs", "fn main() {}")
+        .build();
+
+    p.cargo("doc").masquerade_as_nightly_cargo().run();
+
+    println!("{}", p.root().display());
 }
 
 #[cargo_test]
@@ -2527,37 +2550,54 @@ fn scrape_examples_crate_with_dash() {
         .file("examples/a.rs", "fn main() { da_sh::foo(); }")
         .build();
 
-    p.cargo("doc -Zunstable-options -Z rustdoc-scrape-examples=all")
-        .masquerade_as_nightly_cargo()
-        .run();
+    p.cargo("doc").masquerade_as_nightly_cargo().run();
 
     let doc_html = p.read_file("target/doc/da_sh/fn.foo.html");
     assert!(doc_html.contains("Examples found in repository"));
 }
 
 #[cargo_test]
-fn scrape_examples_missing_flag() {
+fn scrape_examples_configure_target() {
     if !is_nightly() {
+        // -Z rustdoc-scrape-examples is unstable
         return;
     }
 
     let p = project()
         .file(
             "Cargo.toml",
             r#"
-            [package]
-            name = "foo"
-            version = "1.2.4"
-            authors = []
-        "#,
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                authors = []
+
+                [lib]
+                doc-scrape-examples = false
+
+                [[test]]
+                name = "a_test"
+                doc-scrape-examples = true
+            "#,
+        )
+        .file(
+            "src/lib.rs",
+            "pub fn foo() {} fn lib_must_not_appear() { foo(); }",
+        )
+        .file("examples/a.rs", "fn example_must_appear() { foo::foo(); }")
+        .file(
+            "tests/a_test.rs",
+            "#[test] fn test_must_appear() { foo::foo(); }",
         )
-        .file("src/lib.rs", "//! These are the docs!")
         .build();
-    p.cargo("doc -Zrustdoc-scrape-examples")
-        .masquerade_as_nightly_cargo()
-        .with_status(101)
-        .with_stderr("error: -Z rustdoc-scrape-examples must take [..] an argument")
-        .run();
+
+    p.cargo("doc").masquerade_as_nightly_cargo().run();
+
+    let doc_html = p.read_file("target/doc/foo/fn.foo.html");
+    println!("{}", doc_html);
+    assert!(!doc_html.contains("lib_must_not_appear"));
+    assert!(doc_html.contains("example_must_appear"));
+    assert!(doc_html.contains("test_must_appear"));
 }
 
 #[cargo_test]