From 2f75b4e7b0e362f5947ab015bc7a8b10702756d8 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 11 Jun 2025 07:48:23 +0200 Subject: [PATCH] Add `DocJson` profile for benchmarking JSON rustdoc output --- collector/README.md | 8 +++---- .../bitmaps-3.2.1-new-solver/perf-config.json | 3 ++- .../perf-config.json | 3 ++- .../perf-config.json | 3 ++- .../serde-1.0.219-new-solver/perf-config.json | 3 ++- .../syn-2.0.101-new-solver/perf-config.json | 3 ++- collector/src/bin/collector.rs | 2 +- collector/src/compile/benchmark/mod.rs | 2 +- collector/src/compile/benchmark/profile.rs | 15 ++++++++++++ collector/src/compile/execute/bencher.rs | 1 + collector/src/compile/execute/mod.rs | 23 ++++++++++++++----- collector/src/toolchain.rs | 6 ++--- database/src/lib.rs | 4 ++++ docs/glossary.md | 4 +++- site/src/request_handlers/dashboard.rs | 3 +++ 15 files changed, 62 insertions(+), 21 deletions(-) diff --git a/collector/README.md b/collector/README.md index 61c79aba3..3dc9f1694 100644 --- a/collector/README.md +++ b/collector/README.md @@ -146,11 +146,11 @@ The following options alter the behaviour of the `bench_local` subcommand. possible choices are one or more (comma-separated) of `Primary`, `Secondary`, `Stable`, and `All`. The default is `Primary,Secondary`. - `--profiles `: the profiles to be benchmarked. The possible choices - are one or more (comma-separated) of `Check`, `Debug`, `Doc`, `Opt`, and - `All`. The default is `Check,Debug,Opt`. + are one or more (comma-separated) of `Check`, `Debug`, `Doc`, `DocJson`, `Opt`, + `Clippy` and `All`. The default is `Check,Debug,Opt`. - `--rustdoc `: a path (relative or absolute) to a rustdoc - executable that will be benchmarked (but only if a `Doc` profile is requested - with `--profiles`). If a `Doc` profile is requested, by default the tool will + executable that will be benchmarked (but only if a `Doc`/`DocJson` profile is requested + with `--profiles`). If a `Doc`/`DocJson` profile is requested, by default the tool will look for a rustdoc executable next to the rustc specified via the `` argument. - `--scenarios `: the scenarios to be benchmarked. The possible diff --git a/collector/compile-benchmarks/bitmaps-3.2.1-new-solver/perf-config.json b/collector/compile-benchmarks/bitmaps-3.2.1-new-solver/perf-config.json index 11a19bc0e..94b2624ed 100644 --- a/collector/compile-benchmarks/bitmaps-3.2.1-new-solver/perf-config.json +++ b/collector/compile-benchmarks/bitmaps-3.2.1-new-solver/perf-config.json @@ -3,7 +3,8 @@ "category": "secondary", "cargo_rustc_opts": "-Znext-solver=globally", "excluded_profiles": [ - "Doc" + "Doc", + "DocJson" ], "excluded_scenarios": [ "IncrFull", diff --git a/collector/compile-benchmarks/html5ever-0.31.0-new-solver/perf-config.json b/collector/compile-benchmarks/html5ever-0.31.0-new-solver/perf-config.json index 03ed029cd..9b64033b6 100644 --- a/collector/compile-benchmarks/html5ever-0.31.0-new-solver/perf-config.json +++ b/collector/compile-benchmarks/html5ever-0.31.0-new-solver/perf-config.json @@ -4,7 +4,8 @@ "category": "secondary", "cargo_rustc_opts": "-Znext-solver=globally", "excluded_profiles": [ - "Doc" + "Doc", + "DocJson" ], "excluded_scenarios": [ "IncrFull", diff --git a/collector/compile-benchmarks/nalgebra-0.33.0-new-solver/perf-config.json b/collector/compile-benchmarks/nalgebra-0.33.0-new-solver/perf-config.json index 11a19bc0e..94b2624ed 100644 --- a/collector/compile-benchmarks/nalgebra-0.33.0-new-solver/perf-config.json +++ b/collector/compile-benchmarks/nalgebra-0.33.0-new-solver/perf-config.json @@ -3,7 +3,8 @@ "category": "secondary", "cargo_rustc_opts": "-Znext-solver=globally", "excluded_profiles": [ - "Doc" + "Doc", + "DocJson" ], "excluded_scenarios": [ "IncrFull", diff --git a/collector/compile-benchmarks/serde-1.0.219-new-solver/perf-config.json b/collector/compile-benchmarks/serde-1.0.219-new-solver/perf-config.json index 11a19bc0e..94b2624ed 100644 --- a/collector/compile-benchmarks/serde-1.0.219-new-solver/perf-config.json +++ b/collector/compile-benchmarks/serde-1.0.219-new-solver/perf-config.json @@ -3,7 +3,8 @@ "category": "secondary", "cargo_rustc_opts": "-Znext-solver=globally", "excluded_profiles": [ - "Doc" + "Doc", + "DocJson" ], "excluded_scenarios": [ "IncrFull", diff --git a/collector/compile-benchmarks/syn-2.0.101-new-solver/perf-config.json b/collector/compile-benchmarks/syn-2.0.101-new-solver/perf-config.json index 11a19bc0e..94b2624ed 100644 --- a/collector/compile-benchmarks/syn-2.0.101-new-solver/perf-config.json +++ b/collector/compile-benchmarks/syn-2.0.101-new-solver/perf-config.json @@ -3,7 +3,8 @@ "category": "secondary", "cargo_rustc_opts": "-Znext-solver=globally", "excluded_profiles": [ - "Doc" + "Doc", + "DocJson" ], "excluded_scenarios": [ "IncrFull", diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index 16f1078db..26e0f0936 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -151,7 +151,7 @@ fn generate_diffs( for benchmark in benchmarks { for &profile in profiles { for scenario in scenarios.iter().flat_map(|scenario| { - if profile == Profile::Doc && scenario.is_incr() { + if profile.is_doc() && scenario.is_incr() { return vec![]; } match scenario { diff --git a/collector/src/compile/benchmark/mod.rs b/collector/src/compile/benchmark/mod.rs index 2662e818e..f8115ce10 100644 --- a/collector/src/compile/benchmark/mod.rs +++ b/collector/src/compile/benchmark/mod.rs @@ -400,7 +400,7 @@ impl Benchmark { } // Rustdoc does not support incremental compilation - if profile != Profile::Doc { + if !profile.is_doc() { // An incremental from scratch (slowest incremental case). // This is required for any subsequent incremental builds. if scenarios.iter().any(|s| s.is_incr()) { diff --git a/collector/src/compile/benchmark/profile.rs b/collector/src/compile/benchmark/profile.rs index aea8046a1..0946905e0 100644 --- a/collector/src/compile/benchmark/profile.rs +++ b/collector/src/compile/benchmark/profile.rs @@ -8,10 +8,17 @@ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, clap::ValueEnum, serde::Deserialize)] #[value(rename_all = "PascalCase")] pub enum Profile { + /// Perform the equivalent of `cargo check`. Check, + /// Perform the equivalent of `cargo build`. Debug, + /// Perform the equivalent of `cargo doc`. Doc, + /// Perform the equivalent of `cargo doc` with `--output-format=json`. + DocJson, + /// Perform the equivalent of `cargo build --release`. Opt, + /// Perform the equivalent of `cargo clippy`. Clippy, } @@ -21,8 +28,16 @@ impl Profile { Profile::Check, Profile::Debug, Profile::Doc, + Profile::DocJson, Profile::Opt, Profile::Clippy, ] } + + pub fn is_doc(&self) -> bool { + match self { + Profile::Doc | Profile::DocJson => true, + Profile::Check | Profile::Debug | Profile::Opt | Profile::Clippy => false, + } + } } diff --git a/collector/src/compile/execute/bencher.rs b/collector/src/compile/execute/bencher.rs index d113b9025..694d78571 100644 --- a/collector/src/compile/execute/bencher.rs +++ b/collector/src/compile/execute/bencher.rs @@ -184,6 +184,7 @@ impl Processor for BenchProcessor<'_> { Profile::Check => database::Profile::Check, Profile::Debug => database::Profile::Debug, Profile::Doc => database::Profile::Doc, + Profile::DocJson => database::Profile::DocJson, Profile::Opt => database::Profile::Opt, Profile::Clippy => database::Profile::Clippy, }; diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs index 64892a2a4..f3ef52d95 100644 --- a/collector/src/compile/execute/mod.rs +++ b/collector/src/compile/execute/mod.rs @@ -67,7 +67,7 @@ impl PerfTool { | ProfileTool(DepGraph) | ProfileTool(MonoItems) | ProfileTool(LlvmIr) => { - if profile == Profile::Doc { + if profile.is_doc() { Some("rustdoc") } else { Some("rustc") @@ -75,7 +75,7 @@ impl PerfTool { } ProfileTool(LlvmLines) => match profile { Profile::Debug | Profile::Opt => Some("llvm-lines"), - Profile::Check | Profile::Doc | Profile::Clippy => None, + Profile::Check | Profile::Doc | Profile::DocJson | Profile::Clippy => None, }, } } @@ -320,11 +320,10 @@ impl<'a> CargoProcess<'a> { } Some(sub) => sub, } + } else if self.profile.is_doc() { + "rustdoc" } else { - match self.profile { - Profile::Doc => "rustdoc", - _ => "rustc", - } + "rustc" }; let mut cmd = self.base_command(self.cwd, cargo_subcommand); @@ -335,6 +334,11 @@ impl<'a> CargoProcess<'a> { } Profile::Debug => {} Profile::Doc => {} + Profile::DocJson => { + // Enable JSON output + cmd.arg("-Zunstable-options"); + cmd.arg("--output-format=json"); + } Profile::Opt => { cmd.arg("--release"); } @@ -369,6 +373,13 @@ impl<'a> CargoProcess<'a> { cmd.env("RUSTC", &*FAKE_CLIPPY); } + if let Profile::DocJson = self.profile { + // Document more things to stress the doc JSON machinery. + // And this is what `cargo-semver-checks` does. + cmd.arg("--document-private-items"); + cmd.arg("--document-hidden-items"); + } + let processor = self .processor_etc .as_mut() diff --git a/collector/src/toolchain.rs b/collector/src/toolchain.rs index 8adf127e3..82ee3e808 100644 --- a/collector/src/toolchain.rs +++ b/collector/src/toolchain.rs @@ -460,15 +460,15 @@ pub fn get_local_toolchain( Some(rustdoc.canonicalize().with_context(|| { format!("failed to canonicalize rustdoc executable {:?}", rustdoc) })?) - } else if profiles.contains(&Profile::Doc) { + } else if profiles.iter().any(|p| p.is_doc()) { // We need a `rustdoc`. Look for one next to `rustc`. if let Ok(rustdoc) = rustc.with_file_name("rustdoc").canonicalize() { debug!("found rustdoc: {:?}", &rustdoc); Some(rustdoc) } else { anyhow::bail!( - "'Doc' build specified but '--rustdoc' not specified and no 'rustdoc' found \ - next to 'rustc'" + "'Doc' or 'DocJson' build specified but '--rustdoc' not specified and no \ + 'rustdoc' found next to 'rustc'" ); } } else { diff --git a/database/src/lib.rs b/database/src/lib.rs index 41d664776..7ff641aa1 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -210,6 +210,8 @@ pub enum Profile { Debug, /// A doc build Doc, + /// A doc build with `--output-format=json` option. + DocJson, /// An optimized "release" build Opt, /// A Clippy run @@ -223,6 +225,7 @@ impl Profile { Profile::Opt => "opt", Profile::Debug => "debug", Profile::Doc => "doc", + Profile::DocJson => "doc-json", Profile::Clippy => "clippy", } } @@ -235,6 +238,7 @@ impl std::str::FromStr for Profile { "check" => Profile::Check, "debug" => Profile::Debug, "doc" => Profile::Doc, + "doc-json" => Profile::DocJson, "opt" => Profile::Opt, "clippy" => Profile::Clippy, _ => return Err(format!("{} is not a profile", s)), diff --git a/docs/glossary.md b/docs/glossary.md index 397dbcfc0..ed6c5f665 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -15,7 +15,9 @@ The following is a glossary of domain specific terminology. Although benchmarks - `check` corresponds to running `cargo check`. - `debug` corresponds to running `cargo build`. - `opt` corresponds to running `cargo build --release`. - - `doc` corresponds to running rustdoc. + - `doc` corresponds to running rustdoc with the JSON output format. + - `doc-json` corresponds to running rustdoc. + - `clippy` corresponds to running `cargo clippy`. * **scenario**: describes the incremental cache state and an optional change in the source since last compilation. - `full`: incremental compilation is not used. - `incr-full`: incremental compilation is used, with an empty incremental cache. diff --git a/site/src/request_handlers/dashboard.rs b/site/src/request_handlers/dashboard.rs index 8ed8c8cd5..1abad8c75 100644 --- a/site/src/request_handlers/dashboard.rs +++ b/site/src/request_handlers/dashboard.rs @@ -174,6 +174,7 @@ pub struct ByProfile { pub check: T, pub debug: T, pub doc: T, + pub doc_json: T, pub opt: T, pub clippy: T, } @@ -188,6 +189,7 @@ impl ByProfile { check: f(Profile::Check).await?, debug: f(Profile::Debug).await?, doc: f(Profile::Doc).await?, + doc_json: f(Profile::DocJson).await?, opt: f(Profile::Opt).await?, clippy: f(Profile::Clippy).await?, }) @@ -201,6 +203,7 @@ impl std::ops::Index for ByProfile { Profile::Check => &self.check, Profile::Debug => &self.debug, Profile::Doc => &self.doc, + Profile::DocJson => &self.doc_json, Profile::Opt => &self.opt, Profile::Clippy => &self.clippy, }