Merge pull request #1630 from Kobzol/runtime-bench-published

Perform runtime benchmarks when benchmarking published artifacts

Author: Kobzol

collector/src/bin/collector.rs

@@ -3,7 +3,7 @@ use crate::compile::benchmark::patch::Patch;
 use crate::compile::benchmark::profile::Profile;
 use crate::compile::benchmark::scenario::Scenario;
 use crate::compile::execute::{CargoProcess, Processor};
-use crate::toolchain::Compiler;
+use crate::toolchain::Toolchain;
 use anyhow::{bail, Context};
 use log::debug;
 use std::collections::{HashMap, HashSet};
@@ -143,7 +143,7 @@ impl Benchmark {
 
     fn mk_cargo_process<'a>(
         &'a self,
-        compiler: Compiler<'a>,
+        toolchain: &'a Toolchain,
         cwd: &'a Path,
         profile: Profile,
     ) -> CargoProcess<'a> {
@@ -163,7 +163,7 @@ impl Benchmark {
         }
 
         CargoProcess {
-            compiler,
+            toolchain,
             processor_name: self.name.clone(),
             cwd,
             profile,
@@ -194,7 +194,7 @@ impl Benchmark {
         processor: &mut dyn Processor,
         profiles: &[Profile],
         scenarios: &[Scenario],
-        compiler: Compiler<'_>,
+        toolchain: &Toolchain,
         iterations: Option<usize>,
     ) -> anyhow::Result<()> {
         if self.config.disabled {
@@ -261,7 +261,7 @@ impl Benchmark {
             for (profile, prep_dir) in &profile_dirs {
                 let server = server.clone();
                 let thread = s.spawn::<_, anyhow::Result<()>>(move || {
-                    self.mk_cargo_process(compiler, prep_dir.path(), *profile)
+                    self.mk_cargo_process(toolchain, prep_dir.path(), *profile)
                         .jobserver(server)
                         .run_rustc(false)?;
                     Ok(())
@@ -308,7 +308,7 @@ impl Benchmark {
 
                 // A full non-incremental build.
                 if scenarios.contains(&Scenario::Full) {
-                    self.mk_cargo_process(compiler, cwd, profile)
+                    self.mk_cargo_process(toolchain, cwd, profile)
                         .processor(processor, Scenario::Full, "Full", None)
                         .run_rustc(true)?;
                 }
@@ -318,15 +318,15 @@ impl Benchmark {
                     // An incremental  from scratch (slowest incremental case).
                     // This is required for any subsequent incremental builds.
                     if scenarios.iter().any(|s| s.is_incr()) {
-                        self.mk_cargo_process(compiler, cwd, profile)
+                        self.mk_cargo_process(toolchain, cwd, profile)
                             .incremental(true)
                             .processor(processor, Scenario::IncrFull, "IncrFull", None)
                             .run_rustc(true)?;
                     }
 
                     // An incremental build with no changes (fastest incremental case).
                     if scenarios.contains(&Scenario::IncrUnchanged) {
-                        self.mk_cargo_process(compiler, cwd, profile)
+                        self.mk_cargo_process(toolchain, cwd, profile)
                             .incremental(true)
                             .processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None)
                             .run_rustc(true)?;
@@ -340,7 +340,7 @@ impl Benchmark {
                             // An incremental build with some changes (realistic
                             // incremental case).
                             let scenario_str = format!("IncrPatched{}", i);
-                            self.mk_cargo_process(compiler, cwd, profile)
+                            self.mk_cargo_process(toolchain, cwd, profile)
                                 .incremental(true)
                                 .processor(
                                     processor,

collector/src/compile/benchmark/mod.rs

@@ -6,7 +6,7 @@ use crate::compile::execute::{
     rustc, DeserializeStatError, PerfTool, ProcessOutputData, Processor, Retry, SelfProfile,
     SelfProfileFiles, Stats, Upload,
 };
-use crate::toolchain::Compiler;
+use crate::toolchain::Toolchain;
 use crate::utils::git::get_rustc_perf_commit;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
@@ -160,11 +160,11 @@ impl<'a> BenchProcessor<'a> {
             .block_on(async move { while let Some(()) = buf.next().await {} });
     }
 
-    pub fn measure_rustc(&mut self, compiler: Compiler<'_>) -> anyhow::Result<()> {
+    pub fn measure_rustc(&mut self, toolchain: &Toolchain) -> anyhow::Result<()> {
         rustc::measure(
             self.rt,
             self.conn,
-            compiler,
+            toolchain,
             self.artifact,
             self.artifact_row_id,
         )

collector/src/compile/execute/bencher.rs

@@ -4,7 +4,7 @@ use crate::compile::benchmark::patch::Patch;
 use crate::compile::benchmark::profile::Profile;
 use crate::compile::benchmark::scenario::Scenario;
 use crate::compile::benchmark::BenchmarkName;
-use crate::toolchain::Compiler;
+use crate::toolchain::Toolchain;
 use crate::{command_output, utils};
 use anyhow::Context;
 use bencher::Bencher;
@@ -113,7 +113,7 @@ impl PerfTool {
 }
 
 pub struct CargoProcess<'a> {
-    pub compiler: Compiler<'a>,
+    pub toolchain: &'a Toolchain,
     pub cwd: &'a Path,
     pub profile: Profile,
     pub incremental: bool,
@@ -144,12 +144,12 @@ impl<'a> CargoProcess<'a> {
     }
 
     fn base_command(&self, cwd: &Path, subcommand: &str) -> Command {
-        let mut cmd = Command::new(Path::new(self.compiler.cargo));
+        let mut cmd = Command::new(Path::new(&self.toolchain.cargo));
         cmd
             // Not all cargo invocations (e.g. `cargo clean`) need all of these
             // env vars set, but it doesn't hurt to have them.
             .env("RUSTC", &*FAKE_RUSTC)
-            .env("RUSTC_REAL", self.compiler.rustc)
+            .env("RUSTC_REAL", &self.toolchain.rustc)
             // We separately pass -Cincremental to the leaf crate --
             // CARGO_INCREMENTAL is cached separately for both the leaf crate
             // and any in-tree dependencies, and we don't want that; it wastes
@@ -164,7 +164,7 @@ impl<'a> CargoProcess<'a> {
             .arg("--manifest-path")
             .arg(&self.manifest_path);
 
-        if let Some(r) = &self.compiler.rustdoc {
+        if let Some(r) = &self.toolchain.rustdoc {
             cmd.env("RUSTDOC", &*FAKE_RUSTDOC).env("RUSTDOC_REAL", r);
         }
         cmd

collector/src/compile/execute/mod.rs

@@ -7,7 +7,7 @@
 //! no real reason for us to compile the standard library twice, and it avoids
 //! having to think about how to deduplicate results.
 
-use crate::toolchain::Compiler;
+use crate::toolchain::Toolchain;
 use crate::utils::git::get_rustc_perf_commit;
 use anyhow::Context;
 use database::ArtifactId;
@@ -20,23 +20,23 @@ use tokio::runtime::Runtime;
 pub fn measure(
     rt: &mut Runtime,
     conn: &mut dyn database::Connection,
-    compiler: Compiler<'_>,
+    toolchain: &Toolchain,
     artifact: &database::ArtifactId,
     aid: database::ArtifactIdNumber,
 ) -> anyhow::Result<()> {
     eprintln!("Running rustc");
 
     checkout(artifact).context("checking out rust-lang/rust")?;
 
-    record(rt, conn, compiler, artifact, aid)?;
+    record(rt, conn, toolchain, artifact, aid)?;
 
     Ok(())
 }
 
 fn record(
     rt: &mut Runtime,
     conn: &mut dyn database::Connection,
-    compiler: Compiler<'_>,
+    toolchain: &Toolchain,
     artifact: &database::ArtifactId,
     aid: database::ArtifactIdNumber,
 ) -> anyhow::Result<()> {
@@ -95,9 +95,12 @@ fn record(
     .arg("rust.deny-warnings=false")
     .arg("--set")
     .arg(&format!("build.rustc={}", fake_rustc.to_str().unwrap()))
-    .env("RUSTC_PERF_REAL_RUSTC", compiler.rustc)
+    .env("RUSTC_PERF_REAL_RUSTC", &toolchain.rustc)
     .arg("--set")
-    .arg(&format!("build.cargo={}", compiler.cargo.to_str().unwrap()))
+    .arg(&format!(
+        "build.cargo={}",
+        toolchain.cargo.to_str().unwrap()
+    ))
     .status()
     .context("configuring")?;
     assert!(status.success(), "configure successful");
@@ -111,7 +114,7 @@ fn record(
                     .context("x.py script canonicalize")?,
             )
             .current_dir(checkout)
-            .env("RUSTC_PERF_REAL_RUSTC", compiler.rustc)
+            .env("RUSTC_PERF_REAL_RUSTC", &toolchain.rustc)
             .arg("build")
             .arg("--stage")
             .arg("0")

collector/src/compile/execute/rustc.rs

@@ -1,12 +1,12 @@
-use crate::toolchain::LocalToolchain;
+use crate::toolchain::Toolchain;
 use anyhow::Context;
 use benchlib::benchmark::passes_filter;
 use cargo_metadata::Message;
 use core::option::Option;
 use core::option::Option::Some;
 use core::result::Result::Ok;
 use std::collections::HashMap;
-use std::io::{BufReader, Write};
+use std::io::BufReader;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command, Stdio};
 use tempfile::TempDir;
@@ -66,7 +66,14 @@ pub struct BenchmarkFilter {
 }
 
 impl BenchmarkFilter {
-    pub fn new(exclude: Option<String>, include: Option<String>) -> BenchmarkFilter {
+    pub fn keep_all() -> Self {
+        Self {
+            exclude: None,
+            include: None,
+        }
+    }
+
+    pub fn new(exclude: Option<String>, include: Option<String>) -> Self {
         Self { exclude, include }
     }
 }
@@ -87,8 +94,8 @@ pub enum CargoIsolationMode {
 /// We assume that each binary defines a benchmark suite using `benchlib`.
 /// We then execute each benchmark suite with the `list-benchmarks` command to find out its
 /// benchmark names.
-pub fn create_runtime_benchmark_suite(
-    toolchain: &LocalToolchain,
+pub fn prepare_runtime_benchmark_suite(
+    toolchain: &Toolchain,
     benchmark_dir: &Path,
     isolation_mode: CargoIsolationMode,
 ) -> anyhow::Result<BenchmarkSuite> {
@@ -114,28 +121,24 @@ pub fn create_runtime_benchmark_suite(
 
     let mut groups = Vec::new();
     for (index, benchmark_crate) in benchmark_crates.into_iter().enumerate() {
-        // Show incremental progress
-        print!(
-            "\r{}\rCompiling `{}` ({}/{group_count})",
-            " ".repeat(80),
-            benchmark_crate.name,
+        println!(
+            "Compiling {:<22} ({}/{group_count})",
+            format!("`{}`", benchmark_crate.name),
             index + 1
         );
-        std::io::stdout().flush().unwrap();
 
         let target_dir = temp_dir.as_ref().map(|d| d.path());
 
         let cargo_process = start_cargo_build(toolchain, &benchmark_crate.path, target_dir)
             .with_context(|| {
-                anyhow::anyhow!("Cannot not start compilation of {}", benchmark_crate.name)
+                anyhow::anyhow!("Cannot start compilation of {}", benchmark_crate.name)
             })?;
         let group =
             parse_benchmark_group(cargo_process, &benchmark_crate.name).with_context(|| {
                 anyhow::anyhow!("Cannot compile runtime benchmark {}", benchmark_crate.name)
             })?;
         groups.push(group);
     }
-    println!();
 
     groups.sort_unstable_by(|a, b| a.binary.cmp(&b.binary));
     log::debug!("Found binaries: {:?}", groups);
@@ -233,7 +236,7 @@ fn parse_benchmark_group(
 /// Starts the compilation of a single runtime benchmark crate.
 /// Returns the stdout output stream of Cargo.
 fn start_cargo_build(
-    toolchain: &LocalToolchain,
+    toolchain: &Toolchain,
     benchmark_dir: &Path,
     target_dir: Option<&Path>,
 ) -> anyhow::Result<Child> {

collector/src/runtime/benchmark.rs

@@ -6,7 +6,7 @@ use thousands::Separable;
 
 use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 pub use benchmark::{
-    create_runtime_benchmark_suite, runtime_benchmark_dir, BenchmarkFilter, BenchmarkGroup,
+    prepare_runtime_benchmark_suite, runtime_benchmark_dir, BenchmarkFilter, BenchmarkGroup,
     BenchmarkSuite, CargoIsolationMode,
 };
 use database::{ArtifactIdNumber, CollectionId, Connection};
@@ -16,6 +16,8 @@ use crate::CollectorCtx;
 
 mod benchmark;
 
+pub const DEFAULT_RUNTIME_ITERATIONS: u32 = 5;
+
 /// Perform a series of runtime benchmarks using the provided `rustc` compiler.
 /// The runtime benchmarks are looked up in `benchmark_dir`, which is expected to be a path
 /// to a Cargo crate. All binaries built by that crate are expected to be runtime benchmark