Rollup merge of #127002 - Kobzol:bootstrap-perf-tool, r=onur-ozkan

matthiaskrgr · web-flow · commit 6d74ffd2381b · 2024-06-29T22:10:56.000+02:00
Implement `x perf` as a separate tool Continues work from #126318, adds a CLI for running `rustc-perf` profiling commands through a new `rustc-perf-wrapper` tool. The CLI is in a separate tool to enable experimentation outside of `bootstrap`. This is probably most of what we can do so far, I'll add support for benchmarking once `rustc-perf` gets a terminal output for comparing benchmark results. r? ``@onur-ozkan``
diff --git a/Cargo.lock b/Cargo.lock
@@ -3461,6 +3461,13 @@ dependencies = [
  "stable_mir",
 ]
 
+[[package]]
+name = "rustc-perf-wrapper"
+version = "0.1.0"
+dependencies = [
+ "clap",
+]
+
 [[package]]
 name = "rustc-rayon"
 version = "0.5.0"
diff --git a/Cargo.toml b/Cargo.toml
@@ -44,6 +44,7 @@ members = [
   "src/tools/rustdoc-gui-test",
   "src/tools/opt-dist",
   "src/tools/coverage-dump",
+  "src/tools/rustc-perf-wrapper",
 ]
 
 exclude = [
diff --git a/src/bootstrap/src/core/build_steps/perf.rs b/src/bootstrap/src/core/build_steps/perf.rs
@@ -1,7 +1,5 @@
-use std::process::Command;
-
 use crate::core::build_steps::compile::{Std, Sysroot};
-use crate::core::build_steps::tool::RustcPerf;
+use crate::core::build_steps::tool::{RustcPerf, Tool};
 use crate::core::builder::Builder;
 use crate::core::config::DebuginfoLevel;
 
@@ -22,24 +20,16 @@ Consider setting `rust.debuginfo-level = 1` in `config.toml`."#);
     let sysroot = builder.ensure(Sysroot::new(compiler));
     let rustc = sysroot.join("bin/rustc");
 
-    let results_dir = builder.build.tempdir().join("rustc-perf");
-
-    let mut cmd = Command::new(collector);
-    let cmd = cmd
-        .arg("profile_local")
-        .arg("eprintln")
-        .arg("--out-dir")
-        .arg(&results_dir)
-        .arg("--include")
-        .arg("helloworld")
-        .arg(&rustc);
-
-    builder.info(&format!("Running `rustc-perf` using `{}`", rustc.display()));
+    let rustc_perf_dir = builder.build.tempdir().join("rustc-perf");
+    let profile_results_dir = rustc_perf_dir.join("results");
 
-    // We need to set the working directory to `src/tools/perf`, so that it can find the directory
-    // with compile-time benchmarks.
-    let cmd = cmd.current_dir(builder.src.join("src/tools/rustc-perf"));
-    builder.build.run(cmd);
+    // We need to take args passed after `--` and pass them to `rustc-perf-wrapper`
+    let args = std::env::args().skip_while(|a| a != "--").skip(1);
 
-    builder.info(&format!("You can find the results at `{}`", results_dir.display()));
+    let mut cmd = builder.tool_cmd(Tool::RustcPerfWrapper);
+    cmd.env("RUSTC_REAL", rustc)
+        .env("PERF_COLLECTOR", collector)
+        .env("PERF_RESULT_DIR", profile_results_dir)
+        .args(args);
+    builder.run(&mut cmd);
 }
diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs
@@ -337,6 +337,7 @@ bootstrap_tool!(
     GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
     RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
     CoverageDump, "src/tools/coverage-dump", "coverage-dump";
+    RustcPerfWrapper, "src/tools/rustc-perf-wrapper", "rustc-perf-wrapper";
 );
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
@@ -470,7 +470,9 @@ Arguments:
         versioned_dirs: bool,
     },
     /// Perform profiling and benchmarking of the compiler using the
-    /// `rustc-perf` benchmark suite.
+    /// `rustc-perf-wrapper` tool.
+    ///
+    /// You need to pass arguments after `--`, e.g.`x perf -- cachegrind`.
     Perf {},
 }
 
diff --git a/src/etc/completions/x.py.fish b/src/etc/completions/x.py.fish
@@ -48,7 +48,7 @@ complete -c x.py -n "__fish_use_subcommand" -f -a "run" -d 'Run tools contained
 complete -c x.py -n "__fish_use_subcommand" -f -a "setup" -d 'Set up the environment for development'
 complete -c x.py -n "__fish_use_subcommand" -f -a "suggest" -d 'Suggest a subset of tests to run, based on modified files'
 complete -c x.py -n "__fish_use_subcommand" -f -a "vendor" -d 'Vendor dependencies'
-complete -c x.py -n "__fish_use_subcommand" -f -a "perf" -d 'Perform profiling and benchmarking of the compiler using the `rustc-perf` benchmark suite'
+complete -c x.py -n "__fish_use_subcommand" -f -a "perf" -d 'Perform profiling and benchmarking of the compiler using the `rustc-perf-wrapper` tool'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build -d 'build target of the stage0 compiler' -r -f
diff --git a/src/etc/completions/x.py.ps1 b/src/etc/completions/x.py.ps1
@@ -75,7 +75,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('setup', 'setup', [CompletionResultType]::ParameterValue, 'Set up the environment for development')
             [CompletionResult]::new('suggest', 'suggest', [CompletionResultType]::ParameterValue, 'Suggest a subset of tests to run, based on modified files')
             [CompletionResult]::new('vendor', 'vendor', [CompletionResultType]::ParameterValue, 'Vendor dependencies')
-            [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf` benchmark suite')
+            [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf-wrapper` tool')
             break
         }
         'x.py;build' {
diff --git a/src/etc/completions/x.py.zsh b/src/etc/completions/x.py.zsh
@@ -856,7 +856,7 @@ _x.py_commands() {
 'setup:Set up the environment for development' \
 'suggest:Suggest a subset of tests to run, based on modified files' \
 'vendor:Vendor dependencies' \
-'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf\` benchmark suite' \
+'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf-wrapper\` tool' \
     )
     _describe -t commands 'x.py commands' commands "$@"
 }
diff --git a/src/tools/rustc-perf-wrapper/Cargo.toml b/src/tools/rustc-perf-wrapper/Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "rustc-perf-wrapper"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+clap = { version = "4.5.7", features = ["derive", "env"] }
diff --git a/src/tools/rustc-perf-wrapper/README.md b/src/tools/rustc-perf-wrapper/README.md
@@ -0,0 +1,3 @@
+# rustc-perf wrapper
+Utility tool for invoking [`rustc-perf`](https://github.com/rust-lang/rustc-perf) for benchmarking/profiling
+a stage1/2 compiler built by bootstrap using `x perf -- <command>`.
diff --git a/src/tools/rustc-perf-wrapper/src/config.rs b/src/tools/rustc-perf-wrapper/src/config.rs
@@ -0,0 +1,45 @@
+use std::fmt::{Display, Formatter};
+
+#[derive(Clone, Copy, Debug, clap::ValueEnum)]
+#[value(rename_all = "PascalCase")]
+pub enum Profile {
+    Check,
+    Debug,
+    Doc,
+    Opt,
+    Clippy,
+}
+
+impl Display for Profile {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Profile::Check => "Check",
+            Profile::Debug => "Debug",
+            Profile::Doc => "Doc",
+            Profile::Opt => "Opt",
+            Profile::Clippy => "Clippy",
+        };
+        f.write_str(name)
+    }
+}
+
+#[derive(Clone, Copy, Debug, clap::ValueEnum)]
+#[value(rename_all = "PascalCase")]
+pub enum Scenario {
+    Full,
+    IncrFull,
+    IncrUnchanged,
+    IncrPatched,
+}
+
+impl Display for Scenario {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Scenario::Full => "Full",
+            Scenario::IncrFull => "IncrFull",
+            Scenario::IncrUnchanged => "IncrUnchanged",
+            Scenario::IncrPatched => "IncrPatched",
+        };
+        f.write_str(name)
+    }
+}
diff --git a/src/tools/rustc-perf-wrapper/src/main.rs b/src/tools/rustc-perf-wrapper/src/main.rs
@@ -0,0 +1,130 @@
+use crate::config::{Profile, Scenario};
+use clap::Parser;
+use std::path::PathBuf;
+use std::process::Command;
+
+mod config;
+
+/// Performs profiling or benchmarking with [`rustc-perf`](https://github.com/rust-lang/rustc-perf)
+/// using a locally built compiler.
+#[derive(Debug, clap::Parser)]
+// Hide arguments from BuildContext in the default usage string.
+// Clap does not seem to have a way of disabling the usage of these arguments.
+#[clap(override_usage = "rustc-perf-wrapper [OPTIONS] <COMMAND>")]
+pub struct Args {
+    #[clap(subcommand)]
+    cmd: PerfCommand,
+
+    #[clap(flatten)]
+    opts: SharedOpts,
+
+    #[clap(flatten)]
+    ctx: BuildContext,
+}
+
+#[derive(Debug, clap::Parser)]
+enum PerfCommand {
+    /// Run `profile_local eprintln`.
+    /// This executes the compiler on the given benchmarks and stores its stderr output.
+    Eprintln,
+    /// Run `profile_local samply`
+    /// This executes the compiler on the given benchmarks and profiles it with `samply`.
+    /// You need to install `samply`, e.g. using `cargo install samply`.
+    Samply,
+    /// Run `profile_local cachegrind`.
+    /// This executes the compiler on the given benchmarks under `Cachegrind`.
+    Cachegrind,
+}
+
+impl PerfCommand {
+    fn is_profiling(&self) -> bool {
+        match self {
+            PerfCommand::Eprintln | PerfCommand::Samply | PerfCommand::Cachegrind => true,
+        }
+    }
+}
+
+#[derive(Debug, clap::Parser)]
+struct SharedOpts {
+    /// Select the benchmarks that you want to run (separated by commas).
+    /// If unspecified, all benchmarks will be executed.
+    #[clap(long, global = true, value_delimiter = ',')]
+    include: Vec<String>,
+    /// Select the scenarios that should be benchmarked.
+    #[clap(
+        long,
+        global = true,
+        value_delimiter = ',',
+        default_value = "Full,IncrFull,IncrUnchanged,IncrPatched"
+    )]
+    scenarios: Vec<Scenario>,
+    /// Select the profiles that should be benchmarked.
+    #[clap(long, global = true, value_delimiter = ',', default_value = "Check,Debug,Opt")]
+    profiles: Vec<Profile>,
+}
+
+/// These arguments are mostly designed to be passed from bootstrap, not by users
+/// directly.
+#[derive(Debug, clap::Parser)]
+struct BuildContext {
+    /// Compiler binary that will be benchmarked/profiled.
+    #[clap(long, hide = true, env = "RUSTC_REAL")]
+    compiler: PathBuf,
+    /// rustc-perf collector binary that will be used for running benchmarks/profilers.
+    #[clap(long, hide = true, env = "PERF_COLLECTOR")]
+    collector: PathBuf,
+    /// Directory where to store results.
+    #[clap(long, hide = true, env = "PERF_RESULT_DIR")]
+    results_dir: PathBuf,
+}
+
+fn main() {
+    let args = Args::parse();
+    run(args);
+}
+
+fn run(args: Args) {
+    let mut cmd = Command::new(args.ctx.collector);
+    match &args.cmd {
+        PerfCommand::Eprintln => {
+            cmd.arg("profile_local").arg("eprintln");
+        }
+        PerfCommand::Samply => {
+            cmd.arg("profile_local").arg("samply");
+        }
+        PerfCommand::Cachegrind => {
+            cmd.arg("profile_local").arg("cachegrind");
+        }
+    }
+    if args.cmd.is_profiling() {
+        cmd.arg("--out-dir").arg(&args.ctx.results_dir);
+    }
+
+    if !args.opts.include.is_empty() {
+        cmd.arg("--include").arg(args.opts.include.join(","));
+    }
+    if !args.opts.profiles.is_empty() {
+        cmd.arg("--profiles")
+            .arg(args.opts.profiles.iter().map(|p| p.to_string()).collect::<Vec<_>>().join(","));
+    }
+    if !args.opts.scenarios.is_empty() {
+        cmd.arg("--scenarios")
+            .arg(args.opts.scenarios.iter().map(|p| p.to_string()).collect::<Vec<_>>().join(","));
+    }
+    cmd.arg(&args.ctx.compiler);
+
+    println!("Running `rustc-perf` using `{}`", args.ctx.compiler.display());
+
+    const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+
+    let rustc_perf_dir = PathBuf::from(MANIFEST_DIR).join("../rustc-perf");
+
+    // We need to set the working directory to `src/tools/perf`, so that it can find the directory
+    // with compile-time benchmarks.
+    let cmd = cmd.current_dir(rustc_perf_dir);
+    cmd.status().expect("error while running rustc-perf collector");
+
+    if args.cmd.is_profiling() {
+        println!("You can find the results at `{}`", args.ctx.results_dir.display());
+    }
+}
diff --git a/triagebot.toml b/triagebot.toml
@@ -331,6 +331,7 @@ trigger_files = [
     "src/tools/tidy",
     "src/tools/rustdoc-gui-test",
     "src/tools/libcxx-version",
+    "src/tools/rustc-perf-wrapper",
 ]
 
 [autolabel."T-infra"]

Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,7 @@ members = [`
`44`	`44`	`"src/tools/rustdoc-gui-test",`
`45`	`45`	`"src/tools/opt-dist",`
`46`	`46`	`"src/tools/coverage-dump",`
	`47`	`+ "src/tools/rustc-perf-wrapper",`
`47`	`48`	`]`
`48`	`49`
`49`	`50`	`exclude = [`
Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {`
`75`	`75`	`[CompletionResult]::new('setup', 'setup', [CompletionResultType]::ParameterValue, 'Set up the environment for development')`
`76`	`76`	`[CompletionResult]::new('suggest', 'suggest', [CompletionResultType]::ParameterValue, 'Suggest a subset of tests to run, based on modified files')`
`77`	`77`	`[CompletionResult]::new('vendor', 'vendor', [CompletionResultType]::ParameterValue, 'Vendor dependencies')`
`78`		- [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf` benchmark suite')
	`78`	+ [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf-wrapper` tool')
`79`	`79`	`break`
`80`	`80`	`}`
`81`	`81`	`'x.py;build' {`
Original file line number	Diff line number	Diff line change
`@@ -856,7 +856,7 @@ _x.py_commands() {`
`856`	`856`	`'setup:Set up the environment for development' \`
`857`	`857`	`'suggest:Suggest a subset of tests to run, based on modified files' \`
`858`	`858`	`'vendor:Vendor dependencies' \`
`859`		-'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf\` benchmark suite' \
	`859`	+'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf-wrapper\` tool' \
`860`	`860`	`)`
`861`	`861`	`_describe -t commands 'x.py commands' commands "$@"`
`862`	`862`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# rustc-perf wrapper`
	`2`	+Utility tool for invoking [`rustc-perf`](https://github.com/rust-lang/rustc-perf) for benchmarking/profiling
	`3`	+a stage1/2 compiler built by bootstrap using `x perf -- <command>`.