Merge pull request #2156 from Kobzol/rustdoc-json-perf-check

Add `DocJson` profile for benchmarking JSON rustdoc output

Author: Kobzol

collector/README.md

@@ -146,11 +146,11 @@ The following options alter the behaviour of the `bench_local` subcommand.
   possible choices are one or more (comma-separated) of `Primary`, `Secondary`,
   `Stable`, and `All`. The default is `Primary,Secondary`.
 - `--profiles <PROFILES>`: the profiles to be benchmarked. The possible choices
-  are one or more (comma-separated) of `Check`, `Debug`, `Doc`, `Opt`, and
-  `All`. The default is `Check,Debug,Opt`.
+  are one or more (comma-separated) of `Check`, `Debug`, `Doc`, `DocJson`, `Opt`,
+  `Clippy` and `All`. The default is `Check,Debug,Opt`.
 - `--rustdoc <RUSTDOC>`: a path (relative or absolute) to a rustdoc
-  executable that will be benchmarked (but only if a `Doc` profile is requested
-  with `--profiles`). If a `Doc` profile is requested, by default the tool will
+  executable that will be benchmarked (but only if a `Doc`/`DocJson` profile is requested
+  with `--profiles`). If a `Doc`/`DocJson` profile is requested, by default the tool will
   look for a rustdoc executable next to the rustc specified via the `<RUSTC>`
   argument.
 - `--scenarios <SCENARIOS>`: the scenarios to be benchmarked. The possible

collector/compile-benchmarks/bitmaps-3.2.1-new-solver/perf-config.json

@@ -3,7 +3,8 @@
   "category": "secondary",
   "cargo_rustc_opts": "-Znext-solver=globally",
   "excluded_profiles": [
-    "Doc"
+    "Doc",
+    "DocJson"
   ],
   "excluded_scenarios": [
     "IncrFull",

collector/compile-benchmarks/html5ever-0.31.0-new-solver/perf-config.json

@@ -4,7 +4,8 @@
   "category": "secondary",
   "cargo_rustc_opts": "-Znext-solver=globally",
   "excluded_profiles": [
-    "Doc"
+    "Doc",
+    "DocJson"
   ],
   "excluded_scenarios": [
     "IncrFull",

collector/compile-benchmarks/nalgebra-0.33.0-new-solver/perf-config.json

@@ -3,7 +3,8 @@
   "category": "secondary",
   "cargo_rustc_opts": "-Znext-solver=globally",
   "excluded_profiles": [
-    "Doc"
+    "Doc",
+    "DocJson"
   ],
   "excluded_scenarios": [
     "IncrFull",

collector/compile-benchmarks/serde-1.0.219-new-solver/perf-config.json

@@ -3,7 +3,8 @@
   "category": "secondary",
   "cargo_rustc_opts": "-Znext-solver=globally",
   "excluded_profiles": [
-    "Doc"
+    "Doc",
+    "DocJson"
   ],
   "excluded_scenarios": [
     "IncrFull",

collector/compile-benchmarks/syn-2.0.101-new-solver/perf-config.json

@@ -3,7 +3,8 @@
   "category": "secondary",
   "cargo_rustc_opts": "-Znext-solver=globally",
   "excluded_profiles": [
-    "Doc"
+    "Doc",
+    "DocJson"
   ],
   "excluded_scenarios": [
     "IncrFull",

collector/src/bin/collector.rs

@@ -151,7 +151,7 @@ fn generate_diffs(
     for benchmark in benchmarks {
         for &profile in profiles {
             for scenario in scenarios.iter().flat_map(|scenario| {
-                if profile == Profile::Doc && scenario.is_incr() {
+                if profile.is_doc() && scenario.is_incr() {
                     return vec![];
                 }
                 match scenario {

collector/src/compile/benchmark/mod.rs

@@ -400,7 +400,7 @@ impl Benchmark {
                 }
 
                 // Rustdoc does not support incremental compilation
-                if profile != Profile::Doc {
+                if !profile.is_doc() {
                     // An incremental  from scratch (slowest incremental case).
                     // This is required for any subsequent incremental builds.
                     if scenarios.iter().any(|s| s.is_incr()) {

collector/src/compile/benchmark/profile.rs

@@ -8,10 +8,17 @@
 #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, clap::ValueEnum, serde::Deserialize)]
 #[value(rename_all = "PascalCase")]
 pub enum Profile {
+    /// Perform the equivalent of `cargo check`.
     Check,
+    /// Perform the equivalent of `cargo build`.
     Debug,
+    /// Perform the equivalent of `cargo doc`.
     Doc,
+    /// Perform the equivalent of `cargo doc` with `--output-format=json`.
+    DocJson,
+    /// Perform the equivalent of `cargo build --release`.
     Opt,
+    /// Perform the equivalent of `cargo clippy`.
     Clippy,
 }
 
@@ -21,8 +28,16 @@ impl Profile {
             Profile::Check,
             Profile::Debug,
             Profile::Doc,
+            Profile::DocJson,
             Profile::Opt,
             Profile::Clippy,
         ]
     }
+
+    pub fn is_doc(&self) -> bool {
+        match self {
+            Profile::Doc | Profile::DocJson => true,
+            Profile::Check | Profile::Debug | Profile::Opt | Profile::Clippy => false,
+        }
+    }
 }

collector/src/compile/execute/bencher.rs

@@ -184,6 +184,7 @@ impl Processor for BenchProcessor<'_> {
                         Profile::Check => database::Profile::Check,
                         Profile::Debug => database::Profile::Debug,
                         Profile::Doc => database::Profile::Doc,
+                        Profile::DocJson => database::Profile::DocJson,
                         Profile::Opt => database::Profile::Opt,
                         Profile::Clippy => database::Profile::Clippy,
                     };

collector/src/compile/execute/mod.rs

@@ -67,15 +67,15 @@ impl PerfTool {
             | ProfileTool(DepGraph)
             | ProfileTool(MonoItems)
             | ProfileTool(LlvmIr) => {
-                if profile == Profile::Doc {
+                if profile.is_doc() {
                     Some("rustdoc")
                 } else {
                     Some("rustc")
                 }
             }
             ProfileTool(LlvmLines) => match profile {
                 Profile::Debug | Profile::Opt => Some("llvm-lines"),
-                Profile::Check | Profile::Doc | Profile::Clippy => None,
+                Profile::Check | Profile::Doc | Profile::DocJson | Profile::Clippy => None,
             },
         }
     }
@@ -320,11 +320,10 @@ impl<'a> CargoProcess<'a> {
                         }
                         Some(sub) => sub,
                     }
+                } else if self.profile.is_doc() {
+                    "rustdoc"
                 } else {
-                    match self.profile {
-                        Profile::Doc => "rustdoc",
-                        _ => "rustc",
-                    }
+                    "rustc"
                 };
 
             let mut cmd = self.base_command(self.cwd, cargo_subcommand);
@@ -335,6 +334,11 @@ impl<'a> CargoProcess<'a> {
                 }
                 Profile::Debug => {}
                 Profile::Doc => {}
+                Profile::DocJson => {
+                    // Enable JSON output
+                    cmd.arg("-Zunstable-options");
+                    cmd.arg("--output-format=json");
+                }
                 Profile::Opt => {
                     cmd.arg("--release");
                 }
@@ -369,6 +373,13 @@ impl<'a> CargoProcess<'a> {
                     cmd.env("RUSTC", &*FAKE_CLIPPY);
                 }
 
+                if let Profile::DocJson = self.profile {
+                    // Document more things to stress the doc JSON machinery.
+                    // And this is what `cargo-semver-checks` does.
+                    cmd.arg("--document-private-items");
+                    cmd.arg("--document-hidden-items");
+                }
+
                 let processor = self
                     .processor_etc
                     .as_mut()

collector/src/toolchain.rs

@@ -460,15 +460,15 @@ pub fn get_local_toolchain(
             Some(rustdoc.canonicalize().with_context(|| {
                 format!("failed to canonicalize rustdoc executable {:?}", rustdoc)
             })?)
-        } else if profiles.contains(&Profile::Doc) {
+        } else if profiles.iter().any(|p| p.is_doc()) {
             // We need a `rustdoc`. Look for one next to `rustc`.
             if let Ok(rustdoc) = rustc.with_file_name("rustdoc").canonicalize() {
                 debug!("found rustdoc: {:?}", &rustdoc);
                 Some(rustdoc)
             } else {
                 anyhow::bail!(
-                    "'Doc' build specified but '--rustdoc' not specified and no 'rustdoc' found \
-                    next to 'rustc'"
+                    "'Doc' or 'DocJson' build specified but '--rustdoc' not specified and no \
+                    'rustdoc' found next to 'rustc'"
                 );
             }
         } else {

database/src/lib.rs

@@ -210,6 +210,8 @@ pub enum Profile {
     Debug,
     /// A doc build
     Doc,
+    /// A doc build with `--output-format=json` option.
+    DocJson,
     /// An optimized "release" build
     Opt,
     /// A Clippy run
@@ -223,6 +225,7 @@ impl Profile {
             Profile::Opt => "opt",
             Profile::Debug => "debug",
             Profile::Doc => "doc",
+            Profile::DocJson => "doc-json",
             Profile::Clippy => "clippy",
         }
     }
@@ -235,6 +238,7 @@ impl std::str::FromStr for Profile {
             "check" => Profile::Check,
             "debug" => Profile::Debug,
             "doc" => Profile::Doc,
+            "doc-json" => Profile::DocJson,
             "opt" => Profile::Opt,
             "clippy" => Profile::Clippy,
             _ => return Err(format!("{} is not a profile", s)),

docs/glossary.md

@@ -15,7 +15,9 @@ The following is a glossary of domain specific terminology. Although benchmarks
   - `check` corresponds to running `cargo check`.
   - `debug` corresponds to running `cargo build`.
   - `opt` corresponds to running `cargo build --release`.
-  - `doc` corresponds to running rustdoc. 
+  - `doc` corresponds to running rustdoc with the JSON output format.
+  - `doc-json` corresponds to running rustdoc.
+  - `clippy` corresponds to running `cargo clippy`.
 * **scenario**: describes the incremental cache state and an optional change in the source since last compilation.
   - `full`: incremental compilation is not used.
   - `incr-full`: incremental compilation is used, with an empty incremental cache.

site/src/request_handlers/dashboard.rs

@@ -174,6 +174,7 @@ pub struct ByProfile<T> {
     pub check: T,
     pub debug: T,
     pub doc: T,
+    pub doc_json: T,
     pub opt: T,
     pub clippy: T,
 }
@@ -188,6 +189,7 @@ impl<T> ByProfile<T> {
             check: f(Profile::Check).await?,
             debug: f(Profile::Debug).await?,
             doc: f(Profile::Doc).await?,
+            doc_json: f(Profile::DocJson).await?,
             opt: f(Profile::Opt).await?,
             clippy: f(Profile::Clippy).await?,
         })
@@ -201,6 +203,7 @@ impl<T> std::ops::Index<Profile> for ByProfile<T> {
             Profile::Check => &self.check,
             Profile::Debug => &self.debug,
             Profile::Doc => &self.doc,
+            Profile::DocJson => &self.doc_json,
             Profile::Opt => &self.opt,
             Profile::Clippy => &self.clippy,
         }