Merge pull request #1453 from rust-lang/runtime-cli

Runtime benchmarks CLI and documentation improvements

Author: Kobzol

Cargo.lock

@@ -8,4 +8,4 @@ Additional documentation on running and setting up the frontend and backend can
 be found in the `README` files in the `collector` and `site` directories.
 
 Additional documentation on the benchmark programs can be found in the `README`
-file in the `collector/benchmarks` directory.
+file in the `collector/compile-benchmarks` and `collector/runtime-benchmarks` directories.

README.md

@@ -13,6 +13,7 @@ serde_json = "1.0.83"
 log = "0.4.17"
 env_logger = "0.9.0"
 clap = { version = "3.2", features = ["derive"] }
+libc = "0.2"
 
 [target.'cfg(unix)'.dependencies]
 perf-event = "0.4.7"

collector/benchlib/Cargo.toml

@@ -1,6 +1,7 @@
 use crate::cli::{parse_cli, Args, BenchmarkArgs};
 use crate::measure::benchmark_function;
 use crate::messages::BenchmarkResult;
+use crate::process::raise_process_priority;
 use log::LevelFilter;
 use std::collections::HashMap;
 
@@ -14,9 +15,16 @@ pub fn benchmark_suite<F: FnOnce(&mut BenchmarkSuite)>(define_func: F) {
     suite.run().expect("Benchmark suite has failed");
 }
 
+/// Type-erased function that performs a benchmark.
+struct BenchmarkWrapper {
+    func: Box<dyn Fn() -> anyhow::Result<BenchmarkResult>>,
+}
+
+type BenchmarkMap = HashMap<&'static str, BenchmarkWrapper>;
+
 #[derive(Default)]
 pub struct BenchmarkSuite {
-    benchmarks: HashMap<&'static str, BenchmarkWrapper>,
+    benchmarks: BenchmarkMap,
 }
 
 impl BenchmarkSuite {
@@ -48,36 +56,47 @@ impl BenchmarkSuite {
     /// Execute the benchmark suite. It will parse CLI arguments and decide what to do based on
     /// them.
     pub fn run(self) -> anyhow::Result<()> {
+        raise_process_priority();
+
         let args = parse_cli()?;
         match args {
             Args::Benchmark(args) => {
-                self.run_benchmark(args)?;
+                run_benchmark(args, self.benchmarks)?;
             }
         }
 
         Ok(())
     }
+}
 
-    fn run_benchmark(self, args: BenchmarkArgs) -> anyhow::Result<()> {
-        let mut items: Vec<_> = self.benchmarks.into_iter().collect();
-        items.sort_unstable_by_key(|item| item.0);
+fn run_benchmark(args: BenchmarkArgs, benchmarks: BenchmarkMap) -> anyhow::Result<()> {
+    let mut items: Vec<(&'static str, BenchmarkWrapper)> = benchmarks
+        .into_iter()
+        .filter(|(name, _)| passes_filter(name, args.exclude.as_deref(), args.include.as_deref()))
+        .collect();
+    items.sort_unstable_by_key(|item| item.0);
 
-        let mut results: Vec<BenchmarkResult> = Vec::with_capacity(items.len());
-        for (name, def) in items {
-            for i in 0..args.iterations {
-                let result = (def.func)()?;
-                log::info!("Benchmark (run {i}) `{}` completed: {:?}", name, result);
-                results.push(result);
-            }
+    let mut results: Vec<BenchmarkResult> = Vec::with_capacity(items.len());
+    for (name, def) in items {
+        for i in 0..args.iterations {
+            let result = (def.func)()?;
+            log::info!("Benchmark (run {i}) `{name}` completed: {result:?}");
+            results.push(result);
         }
-
-        println!("{}", serde_json::to_string(&results)?);
-        Ok(())
     }
+
+    println!("{}", serde_json::to_string(&results)?);
+    Ok(())
 }
 
-struct BenchmarkWrapper {
-    func: Box<dyn Fn() -> anyhow::Result<BenchmarkResult>>,
+/// Tests if the name of the benchmark passes through the include and exclude filter flags.
+fn passes_filter(name: &str, exclude: Option<&str>, include: Option<&str>) -> bool {
+    match (exclude, include) {
+        (Some(exclude), Some(include)) => name.starts_with(include) && !name.starts_with(exclude),
+        (None, Some(include)) => name.starts_with(include),
+        (Some(exclude), None) => !name.starts_with(&exclude),
+        (None, None) => true,
+    }
 }
 
 /// Copied from `iai`, so that we don't have to use unstable features.

collector/benchlib/src/benchmark.rs

@@ -11,6 +11,14 @@ pub struct BenchmarkArgs {
     /// How many times should each benchmark be repeated.
     #[clap(long, default_value = "5")]
     pub iterations: u32,
+
+    /// Exclude all benchmarks matching a prefix in this comma-separated list
+    #[clap(long)]
+    pub exclude: Option<String>,
+
+    /// Include only benchmarks matching a prefix in this comma-separated list
+    #[clap(long)]
+    pub include: Option<String>,
 }
 
 pub fn parse_cli() -> anyhow::Result<Args> {

collector/benchlib/src/cli.rs

@@ -1,5 +1,6 @@
-//! This library defines an API for performing benchmarks of Rust code. It is tailored for the
-//! use-case of `rustc-perf`, that's why we don't use e.g. `criterion` or `iai`.
+//! This library defines an API for performing benchmarks of Rust code and various other utilities
+//! for measuring and benchmarking. It is tailored for the use-case of `rustc-perf`, that's why we
+//! don't use e.g. `criterion` or `iai`.
 //!
 //! We want to be able to define short benchmarks in code, measure specific perf. counters and most
 //! importantly, consume the benchmark results in a programmatic way.
@@ -16,3 +17,4 @@ pub mod benchmark;
 mod cli;
 pub mod measure;
 pub mod messages;
+pub mod process;

collector/benchlib/src/lib.rs

@@ -0,0 +1,16 @@
+#[cfg(unix)]
+pub fn raise_process_priority() {
+    unsafe {
+        // Try to reduce jitter in wall time by increasing our priority to the
+        // maximum
+        for i in (1..21).rev() {
+            let r = libc::setpriority(libc::PRIO_PROCESS as _, libc::getpid() as libc::id_t, -i);
+            if r == 0 {
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(windows)]
+pub fn raise_process_priority() {}

collector/benchlib/src/process.rs

@@ -1,13 +1,13 @@
-# The Benchmark Suite
+# The Compile-time Benchmark Suite
 
-This file describes the programs in the benchmark suite and explains why they
+This file describes the programs in the compile-time benchmark suite and explains why they
 were included.
 
 The suite changes over time. Sometimes the code for a benchmark is updated, in
 which case a small suffix will be added (starting with "-2", then "-3", and so
 on.)
 
-There are three categories of benchmarks, **Primary**, **Secondary**, and
+There are three categories of compile-time benchmarks, **Primary**, **Secondary**, and
 **Stable**.
 
 ## Primary
@@ -198,7 +198,7 @@ Rust code being written today.
       applies correctly, e.g. `target/release/collector bench_local +nightly
       --id Test --profiles=Check --scenarios=IncrPatched
       --include=$NEW_BENCHMARK`
-  - Add the new entry to `collector/benchmarks/README.md`.
+  - Add the new entry to `collector/compile-benchmarks/README.md`.
   - `git add` the `Cargo.lock` file, if it's not already part of the
     benchmark's committed code.
     - If the benchmark has a `.gitignore` file that contains `Cargo.lock`,
@@ -232,7 +232,7 @@ Rust code being written today.
 - In the first commit just remove the old code.
   - Do this with `git rm -r` on the directory.
 - In the second commit do everything else.
-  - Remove the entry from `collector/benchmarks/README.md`.
+  - Remove the entry from `collector/compile-benchmarks/README.md`.
   - `git grep` for occurrences of the old benchmark name (e.g. in
     `.github/workflows/ci.yml` or `ci/check-*.sh`) and see if anything needs
     changing... usually not.

collector/compile-benchmarks/README.md

@@ -0,0 +1,9 @@
+# The Runtime Benchmark Suite
+This directory contains various pieces of code for which we measure how fast do they execute
+when they are compiled with a specific version of `rustc`.
+
+The benchmarks are located in crates that are part of the `runtime-benchmarks` workspace. Each crate
+contains a set of benchmarks defined using named closures.
+
+Benchmarks are divided into sub-crates so that some benchmarks can use various versions of dependency
+crates and also so that they are grouped together by a relevant area (e.g. hashmap benchmarks).