Skip to content

Commit 39e6737

Browse files
committed
Change rustdoc-scrape-examples to be a target-level configuration
1 parent b690ab4 commit 39e6737

File tree

18 files changed

+803
-362
lines changed

18 files changed

+803
-362
lines changed

crates/cargo-test-support/src/compare.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ fn substitute_macros(input: &str) -> String {
168168
("[NOTE]", "note:"),
169169
("[HELP]", "help:"),
170170
("[DOCUMENTING]", " Documenting"),
171+
("[SCRAPING]", " Scraping"),
171172
("[FRESH]", " Fresh"),
172173
("[UPDATING]", " Updating"),
173174
("[ADDING]", " Adding"),

src/cargo/core/compiler/context/compilation_files.rs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -451,9 +451,11 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
451451
vec![]
452452
}
453453
CompileMode::Docscrape => {
454-
let path = self
455-
.deps_dir(unit)
456-
.join(format!("{}.examples", unit.buildkey()));
454+
// The file name needs to be stable across Cargo sessions.
455+
// This originally used unit.buildkey(), but that isn't stable,
456+
// so we use metadata instead (prefixed with name for debugging).
457+
let file_name = format!("{}-{}.examples", unit.pkg.name(), self.metadata(unit));
458+
let path = self.deps_dir(unit).join(file_name);
457459
vec![OutputFile {
458460
path,
459461
hardlink: None,

src/cargo/core/compiler/context/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ pub struct Context<'a, 'cfg> {
7777
/// Map of Doc/Docscrape units to metadata for their -Cmetadata flag.
7878
/// See Context::find_metadata_units for more details.
7979
pub metadata_for_doc_units: HashMap<Unit, Metadata>,
80+
81+
pub failed_scrape_units: Arc<Mutex<HashSet<Metadata>>>,
8082
}
8183

8284
impl<'a, 'cfg> Context<'a, 'cfg> {
@@ -115,6 +117,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
115117
rustc_clients: HashMap::new(),
116118
lto: HashMap::new(),
117119
metadata_for_doc_units: HashMap::new(),
120+
failed_scrape_units: Arc::new(Mutex::new(HashSet::new())),
118121
})
119122
}
120123

src/cargo/core/compiler/fingerprint.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1275,7 +1275,7 @@ fn calculate_normal(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Finger
12751275

12761276
// Afterwards calculate our own fingerprint information.
12771277
let target_root = target_root(cx);
1278-
let local = if unit.mode.is_doc() {
1278+
let local = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
12791279
// rustdoc does not have dep-info files.
12801280
let fingerprint = pkg_fingerprint(cx.bcx, &unit.pkg).with_context(|| {
12811281
format!(
@@ -1302,7 +1302,7 @@ fn calculate_normal(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Finger
13021302
// Fill out a bunch more information that we'll be tracking typically
13031303
// hashed to take up less space on disk as we just need to know when things
13041304
// change.
1305-
let extra_flags = if unit.mode.is_doc() {
1305+
let extra_flags = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
13061306
cx.bcx.rustdocflags_args(unit)
13071307
} else {
13081308
cx.bcx.rustflags_args(unit)

src/cargo/core/compiler/job_queue.rs

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ struct DrainState<'cfg> {
134134
active: HashMap<JobId, Unit>,
135135
compiled: HashSet<PackageId>,
136136
documented: HashSet<PackageId>,
137+
scraped: HashSet<PackageId>,
137138
counts: HashMap<PackageId, usize>,
138139
progress: Progress<'cfg>,
139140
next_id: u32,
@@ -353,6 +354,12 @@ enum Message {
353354
diag: String,
354355
fixable: bool,
355356
},
357+
// This is distinct from Diagnostic because it gets emitted through
358+
// a different Shell pathway
359+
Warning {
360+
id: JobId,
361+
warning: String,
362+
},
356363
WarningCount {
357364
id: JobId,
358365
emitted: bool,
@@ -426,6 +433,14 @@ impl<'a, 'cfg> JobState<'a, 'cfg> {
426433
Ok(())
427434
}
428435

436+
pub fn warning(&self, warning: String) -> CargoResult<()> {
437+
self.messages.push_bounded(Message::Warning {
438+
id: self.id,
439+
warning,
440+
});
441+
Ok(())
442+
}
443+
429444
/// A method used to signal to the coordinator thread that the rmeta file
430445
/// for an rlib has been produced. This is only called for some rmeta
431446
/// builds when required, and can be called at any time before a job ends.
@@ -475,8 +490,10 @@ impl<'cfg> JobQueue<'cfg> {
475490
.filter(|dep| {
476491
// Binaries aren't actually needed to *compile* tests, just to run
477492
// them, so we don't include this dependency edge in the job graph.
493+
// But we shouldn't filter out dependencies being scraped for Rustdoc.
478494
(!dep.unit.target.is_test() && !dep.unit.target.is_bin())
479495
|| dep.unit.artifact.is_true()
496+
|| dep.unit.mode.is_doc_scrape()
480497
})
481498
.map(|dep| {
482499
// Handle the case here where our `unit -> dep` dependency may
@@ -563,6 +580,7 @@ impl<'cfg> JobQueue<'cfg> {
563580
active: HashMap::new(),
564581
compiled: HashSet::new(),
565582
documented: HashSet::new(),
583+
scraped: HashSet::new(),
566584
counts: self.counts,
567585
progress,
568586
next_id: 0,
@@ -739,6 +757,10 @@ impl<'cfg> DrainState<'cfg> {
739757
cnts.disallow_fixable();
740758
}
741759
}
760+
Message::Warning { id, warning } => {
761+
cx.bcx.config.shell().warn(warning)?;
762+
self.bump_warning_count(id, true, false);
763+
}
742764
Message::WarningCount {
743765
id,
744766
emitted,
@@ -782,6 +804,16 @@ impl<'cfg> DrainState<'cfg> {
782804
debug!("end ({:?}): {:?}", unit, result);
783805
match result {
784806
Ok(()) => self.finish(id, &unit, artifact, cx)?,
807+
Err(_)
808+
if unit.mode.is_doc_scrape()
809+
&& unit.target.doc_scrape_examples().is_unset() =>
810+
{
811+
cx.failed_scrape_units
812+
.lock()
813+
.unwrap()
814+
.insert(cx.files().metadata(&unit));
815+
self.queue.finish(&unit, &artifact);
816+
}
785817
Err(error) => {
786818
let msg = "The following warnings were emitted during compilation:";
787819
self.emit_warnings(Some(msg), &unit, cx)?;
@@ -1303,8 +1335,11 @@ impl<'cfg> DrainState<'cfg> {
13031335
unit: &Unit,
13041336
fresh: Freshness,
13051337
) -> CargoResult<()> {
1306-
if (self.compiled.contains(&unit.pkg.package_id()) && !unit.mode.is_doc())
1338+
if (self.compiled.contains(&unit.pkg.package_id())
1339+
&& !unit.mode.is_doc()
1340+
&& !unit.mode.is_doc_scrape())
13071341
|| (self.documented.contains(&unit.pkg.package_id()) && unit.mode.is_doc())
1342+
|| (self.scraped.contains(&unit.pkg.package_id()) && unit.mode.is_doc_scrape())
13081343
{
13091344
return Ok(());
13101345
}
@@ -1318,6 +1353,9 @@ impl<'cfg> DrainState<'cfg> {
13181353
config.shell().status("Documenting", &unit.pkg)?;
13191354
} else if unit.mode.is_doc_test() {
13201355
// Skip doc test.
1356+
} else if unit.mode.is_doc_scrape() {
1357+
self.scraped.insert(unit.pkg.package_id());
1358+
config.shell().status("Scraping", &unit.pkg)?;
13211359
} else {
13221360
self.compiled.insert(unit.pkg.package_id());
13231361
if unit.mode.is_check() {

src/cargo/core/compiler/mod.rs

Lines changed: 67 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ mod unit;
2222
pub mod unit_dependencies;
2323
pub mod unit_graph;
2424

25-
use std::collections::HashSet;
25+
use std::collections::{HashMap, HashSet};
2626
use std::env;
2727
use std::ffi::{OsStr, OsString};
2828
use std::fs::{self, File};
@@ -55,7 +55,7 @@ use crate::core::compiler::future_incompat::FutureIncompatReport;
5555
pub use crate::core::compiler::unit::{Unit, UnitInterner};
5656
use crate::core::manifest::TargetSourcePath;
5757
use crate::core::profiles::{PanicStrategy, Profile, Strip};
58-
use crate::core::{Feature, PackageId, Target};
58+
use crate::core::{Feature, PackageId, Target, Verbosity};
5959
use crate::util::errors::{CargoResult, VerboseError};
6060
use crate::util::interning::InternedString;
6161
use crate::util::machine_message::{self, Message};
@@ -654,13 +654,17 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
654654
rustdoc.arg("-C").arg(format!("metadata={}", metadata));
655655

656656
let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
657-
let output_dir = cx.files().deps_dir(unit);
658-
Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
657+
cx.outputs(unit)
658+
.map(|outputs| outputs[0].path.to_path_buf())
659659
};
660660

661661
if unit.mode.is_doc_scrape() {
662662
debug_assert!(cx.bcx.scrape_units.contains(unit));
663663

664+
if unit.target.is_test() {
665+
rustdoc.arg("--scrape-tests");
666+
}
667+
664668
rustdoc.arg("-Zunstable-options");
665669

666670
rustdoc
@@ -678,18 +682,23 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
678682
rustdoc.arg("--scrape-examples-target-crate").arg(name);
679683
}
680684
}
681-
} else if cx.bcx.scrape_units.len() > 0 && cx.bcx.ws.unit_needs_doc_scrape(unit) {
682-
// We only pass scraped examples to packages in the workspace
683-
// since examples are only coming from reverse-dependencies of workspace packages
685+
}
684686

687+
let should_include_scrape_units = unit.mode.is_doc()
688+
&& cx.bcx.scrape_units.len() > 0
689+
&& cx.bcx.ws.unit_needs_doc_scrape(unit);
690+
let scrape_outputs = if should_include_scrape_units {
685691
rustdoc.arg("-Zunstable-options");
686-
687-
for scrape_unit in &cx.bcx.scrape_units {
688-
rustdoc
689-
.arg("--with-examples")
690-
.arg(scrape_output_path(scrape_unit)?);
691-
}
692-
}
692+
Some(
693+
cx.bcx
694+
.scrape_units
695+
.iter()
696+
.map(|unit| Ok((cx.files().metadata(unit), scrape_output_path(unit)?)))
697+
.collect::<CargoResult<HashMap<_, _>>>()?,
698+
)
699+
} else {
700+
None
701+
};
693702

694703
build_deps_args(&mut rustdoc, cx, unit)?;
695704
rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;
@@ -700,19 +709,45 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
700709
append_crate_version_flag(unit, &mut rustdoc);
701710
}
702711

712+
let target_desc = unit.target.description_named();
703713
let name = unit.pkg.name().to_string();
704714
let build_script_outputs = Arc::clone(&cx.build_script_outputs);
705715
let package_id = unit.pkg.package_id();
706716
let manifest_path = PathBuf::from(unit.pkg.manifest_path());
717+
let relative_manifest_path = manifest_path
718+
.strip_prefix(cx.bcx.ws.root())
719+
.unwrap_or(&manifest_path)
720+
.to_owned();
707721
let target = Target::clone(&unit.target);
708722
let mut output_options = OutputOptions::new(cx, unit);
709723
let script_metadata = cx.find_build_script_metadata(unit);
724+
let failed_scrape_units = Arc::clone(&cx.failed_scrape_units);
725+
let hide_diagnostics_for_scrape_unit = unit.mode.is_doc_scrape()
726+
&& unit.target.doc_scrape_examples().is_unset()
727+
&& !matches!(cx.bcx.config.shell().verbosity(), Verbosity::Verbose);
728+
if hide_diagnostics_for_scrape_unit {
729+
output_options.show_diagnostics = false;
730+
}
710731
Ok(Work::new(move |state| {
711732
add_custom_flags(
712733
&mut rustdoc,
713734
&build_script_outputs.lock().unwrap(),
714735
script_metadata,
715736
)?;
737+
738+
// Add the output of scraped examples to the rustdoc command.
739+
// This action must happen after the unit's dependencies have finished,
740+
// because some of those deps may be Docscrape units which have failed.
741+
// So we dynamically determine which `--with-examples` flags to pass here.
742+
if let Some(scrape_outputs) = scrape_outputs {
743+
let failed_scrape_units = failed_scrape_units.lock().unwrap();
744+
for (metadata, output_path) in &scrape_outputs {
745+
if !failed_scrape_units.contains(metadata) {
746+
rustdoc.arg("--with-examples").arg(output_path);
747+
}
748+
}
749+
}
750+
716751
let crate_dir = doc_dir.join(&crate_name);
717752
if crate_dir.exists() {
718753
// Remove output from a previous build. This ensures that stale
@@ -722,7 +757,7 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
722757
}
723758
state.running(&rustdoc);
724759

725-
rustdoc
760+
let result = rustdoc
726761
.exec_with_streaming(
727762
&mut |line| on_stdout_line(state, line, package_id, &target),
728763
&mut |line| {
@@ -737,7 +772,23 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
737772
},
738773
false,
739774
)
740-
.with_context(|| format!("could not document `{}`", name))?;
775+
.with_context(|| format!("could not document `{}`", name));
776+
777+
if let Err(e) = result {
778+
if hide_diagnostics_for_scrape_unit {
779+
let diag = format!(
780+
"\
781+
failed to scan {target_desc} in package `{name}` for example code usage
782+
Try running with `--verbose` to see the error message.
783+
If this example should not be scanned, consider adding `doc-scrape-examples = false` to the `[[example]]` definition in {}",
784+
relative_manifest_path.display()
785+
);
786+
state.warning(diag)?;
787+
}
788+
789+
return Err(e);
790+
}
791+
741792
Ok(())
742793
}))
743794
}

src/cargo/core/compiler/rustdoc.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,20 @@ pub fn add_root_urls(
190190
}
191191
Ok(())
192192
}
193+
194+
#[derive(Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Copy)]
195+
pub enum RustdocScrapeExamples {
196+
Enabled,
197+
Disabled,
198+
Unset,
199+
}
200+
201+
impl RustdocScrapeExamples {
202+
pub fn is_enabled(&self) -> bool {
203+
matches!(self, RustdocScrapeExamples::Enabled)
204+
}
205+
206+
pub fn is_unset(&self) -> bool {
207+
matches!(self, RustdocScrapeExamples::Unset)
208+
}
209+
}

src/cargo/core/compiler/unit_dependencies.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -718,13 +718,14 @@ fn compute_deps_doc(
718718
// Add all units being scraped for examples as a dependency of top-level Doc units.
719719
if state.ws.unit_needs_doc_scrape(unit) {
720720
for scrape_unit in state.scrape_units.iter() {
721-
deps_of(scrape_unit, state, unit_for)?;
721+
let scrape_unit_for = UnitFor::new_normal(scrape_unit.kind);
722+
deps_of(scrape_unit, state, scrape_unit_for)?;
722723
ret.push(new_unit_dep(
723724
state,
724725
scrape_unit,
725726
&scrape_unit.pkg,
726727
&scrape_unit.target,
727-
unit_for,
728+
scrape_unit_for,
728729
scrape_unit.kind,
729730
scrape_unit.mode,
730731
IS_NO_ARTIFACT_DEP,
@@ -1088,7 +1089,6 @@ impl<'a, 'cfg> State<'a, 'cfg> {
10881089
if !dep.is_transitive()
10891090
&& !unit.target.is_test()
10901091
&& !unit.target.is_example()
1091-
&& !unit.mode.is_doc_scrape()
10921092
&& !unit.mode.is_any_test()
10931093
{
10941094
return false;

src/cargo/core/features.rs

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -688,10 +688,8 @@ unstable_cli_options!(
688688
terminal_width: Option<Option<usize>> = ("Provide a terminal width to rustc for error truncation"),
689689
publish_timeout: bool = ("Enable the `publish.timeout` key in .cargo/config.toml file"),
690690
unstable_options: bool = ("Allow the usage of unstable options"),
691-
// TODO(wcrichto): move scrape example configuration into Cargo.toml before stabilization
692-
// See: https://github.com/rust-lang/cargo/pull/9525#discussion_r728470927
693-
rustdoc_scrape_examples: Option<String> = ("Allow rustdoc to scrape examples from reverse-dependencies for documentation"),
694691
skip_rustdoc_fingerprint: bool = (HIDDEN),
692+
rustdoc_scrape_examples: bool = ("Allows Rustdoc to scrape code examples from reverse-dependencies"),
695693
);
696694

697695
const STABILIZED_COMPILE_PROGRESS: &str = "The progress bar is now always \
@@ -961,15 +959,7 @@ impl CliUnstable {
961959
"namespaced-features" => stabilized_warn(k, "1.60", STABILISED_NAMESPACED_FEATURES),
962960
"weak-dep-features" => stabilized_warn(k, "1.60", STABILIZED_WEAK_DEP_FEATURES),
963961
"credential-process" => self.credential_process = parse_empty(k, v)?,
964-
"rustdoc-scrape-examples" => {
965-
if let Some(s) = v {
966-
self.rustdoc_scrape_examples = Some(s.to_string())
967-
} else {
968-
bail!(
969-
r#"-Z rustdoc-scrape-examples must take "all" or "examples" as an argument"#
970-
)
971-
}
972-
}
962+
"rustdoc-scrape-examples" => self.rustdoc_scrape_examples = parse_empty(k, v)?,
973963
"skip-rustdoc-fingerprint" => self.skip_rustdoc_fingerprint = parse_empty(k, v)?,
974964
"compile-progress" => stabilized_warn(k, "1.30", STABILIZED_COMPILE_PROGRESS),
975965
"offline" => stabilized_err(k, "1.36", STABILIZED_OFFLINE)?,

0 commit comments

Comments
 (0)