Change terminology

database -> suite
suite -> group

Author: Kobzol

collector/benchlib/src/benchmark.rs

@@ -5,13 +5,13 @@ use crate::measure::benchmark_function;
 use crate::process::raise_process_priority;
 use std::collections::HashMap;
 
-/// Create a new benchmark suite. Use the closure argument to define benchmarks.
-pub fn benchmark_suite<F: FnOnce(&mut BenchmarkSuite)>(define_func: F) {
+/// Create a new benchmark group. Use the closure argument to define benchmarks.
+pub fn benchmark_group<F: FnOnce(&mut BenchmarkGroup)>(define_func: F) {
     env_logger::init();
 
-    let mut suite = BenchmarkSuite::new();
-    define_func(&mut suite);
-    suite.run().expect("Benchmark suite has failed");
+    let mut group = BenchmarkGroup::new();
+    define_func(&mut group);
+    group.run().expect("Benchmark group execution has failed");
 }
 
 /// Type-erased function that executes a single benchmark.
@@ -22,11 +22,11 @@ struct BenchmarkWrapper {
 type BenchmarkMap = HashMap<&'static str, BenchmarkWrapper>;
 
 #[derive(Default)]
-pub struct BenchmarkSuite {
+pub struct BenchmarkGroup {
     benchmarks: BenchmarkMap,
 }
 
-impl BenchmarkSuite {
+impl BenchmarkGroup {
     pub fn new() -> Self {
         Self::default()
     }
@@ -39,7 +39,7 @@ impl BenchmarkSuite {
         constructor: F,
     ) {
         // We want to type-erase the target `func` by wrapping it in a Box.
-        let benchmark_func = Box::new(move || benchmark_function(name, constructor.clone()));
+        let benchmark_func = Box::new(move || benchmark_function(constructor.clone()));
         let benchmark_def = BenchmarkWrapper {
             func: benchmark_func,
         };
@@ -48,7 +48,7 @@ impl BenchmarkSuite {
         }
     }
 
-    /// Execute the benchmark suite. It will parse CLI arguments and decide what to do based on
+    /// Execute the benchmark group. It will parse CLI arguments and decide what to do based on
     /// them.
     pub fn run(self) -> anyhow::Result<()> {
         raise_process_priority();
@@ -100,19 +100,19 @@ fn run_benchmark(args: BenchmarkArgs, benchmarks: BenchmarkMap) -> anyhow::Resul
     Ok(())
 }
 
-/// Adds a single benchmark to the benchmark suite.
+/// Adds a single benchmark to the benchmark group.
 /// ```ignore
 /// use benchlib::define_benchmark;
 ///
-/// define_benchmark!(suite, my_bench, {
+/// define_benchmark!(group, my_bench, {
 ///     || do_something()
 /// });
 /// ```
 #[macro_export]
 macro_rules! define_benchmark {
-    ($suite:expr, $name:ident, $fun:expr) => {
+    ($group:expr, $name:ident, $fun:expr) => {
         let func = move || $fun;
-        $suite.register(stringify!($name), func);
+        $group.register(stringify!($name), func);
     };
 }
 

collector/benchlib/src/cli.rs

@@ -2,9 +2,9 @@ use clap::{FromArgMatches, IntoApp};
 
 #[derive(clap::Parser, Debug)]
 pub enum Args {
-    /// Benchmark all benchmarks in this benchmark suite and print the results as JSON.
+    /// Benchmark all benchmarks in this benchmark group and print the results as JSON.
     Benchmark(BenchmarkArgs),
-    /// List benchmarks that are defined in the current suite as a JSON array.
+    /// List benchmarks that are defined in the current group as a JSON array.
     ListBenchmarks,
 }
 

collector/benchlib/src/measure/perf_counter/unix.rs

@@ -16,7 +16,6 @@ struct Counters {
 /// The function is executed twice, once to gather wall-time measurement and the second time to
 /// gather perf. counters.
 pub fn benchmark_function<F: Fn() -> Bench + 'static, R, Bench: FnOnce() -> R + 'static>(
-    name: &'static str,
     benchmark_constructor: F,
 ) -> anyhow::Result<BenchmarkStats> {
     let mut group = create_group()?;

collector/runtime-benchmarks/bufreader/src/main.rs

@@ -2,7 +2,7 @@ use std::io::{BufRead, BufReader, Write};
 
 use snap::{read::FrameDecoder, write::FrameEncoder};
 
-use benchlib::benchmark::{benchmark_suite, black_box};
+use benchlib::benchmark::{benchmark_group, black_box};
 use benchlib::define_benchmark;
 
 const BYTES: usize = 64 * 1024 * 1024;
@@ -11,8 +11,8 @@ fn main() {
     // Inspired by https://github.com/rust-lang/rust/issues/102727
     // The pattern we want is a BufReader which wraps a Read impl where one Read::read call will
     // never fill the whole BufReader buffer.
-    benchmark_suite(|suite| {
-        define_benchmark!(suite, bufreader_snappy, {
+    benchmark_group(|group| {
+        define_benchmark!(group, bufreader_snappy, {
             let data = vec![0u8; BYTES];
             move || {
                 let mut compressed = Vec::new();

collector/runtime-benchmarks/hashmap/src/main.rs

@@ -1,11 +1,11 @@
 use benchlib;
-use benchlib::benchmark::benchmark_suite;
+use benchlib::benchmark::benchmark_group;
 use benchlib::define_benchmark;
 
 fn main() {
-    benchmark_suite(|suite| {
+    benchmark_group(|group| {
         // Measures how long does it take to insert 10 thousand numbers into a `hashbrown` hashmap.
-        define_benchmark!(suite, hashmap_insert_10k, {
+        define_benchmark!(group, hashmap_insert_10k, {
             let mut map = hashbrown::HashMap::with_capacity_and_hasher(
                 10000,
                 fxhash::FxBuildHasher::default(),

collector/runtime-benchmarks/nbody/src/main.rs

@@ -1,14 +1,14 @@
 //! Calculates the N-body simulation.
 //! Code taken from https://github.com/prestontw/rust-nbody
 
-use benchlib::benchmark::benchmark_suite;
+use benchlib::benchmark::benchmark_group;
 use benchlib::define_benchmark;
 
 mod nbody;
 
 fn main() {
-    benchmark_suite(|suite| {
-        define_benchmark!(suite, nbody_10k, {
+    benchmark_group(|group| {
+        define_benchmark!(group, nbody_10k, {
             let mut nbody_10k = nbody::init(10000);
             || {
                 for _ in 0..10 {

collector/src/runtime/benchmark.rs

@@ -6,26 +6,26 @@ use core::result::Result::Ok;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-/// A binary that defines several benchmarks using the `benchmark_suite` function from `benchlib`.
+/// A binary that defines several benchmarks using the `benchmark_group` function from `benchlib`.
 #[derive(Debug)]
-pub struct BenchmarkSuite {
+pub struct BenchmarkGroup {
     pub binary: PathBuf,
     pub benchmark_names: Vec<String>,
 }
 
-impl BenchmarkSuite {
+impl BenchmarkGroup {
     pub fn name(&self) -> &str {
         self.binary.file_name().unwrap().to_str().unwrap()
     }
 }
 
 /// A collection of benchmark suites gathered from a directory.
 #[derive(Debug)]
-pub struct BenchmarkDatabase {
-    pub suites: Vec<BenchmarkSuite>,
+pub struct BenchmarkSuite {
+    pub groups: Vec<BenchmarkGroup>,
 }
 
-impl BenchmarkDatabase {
+impl BenchmarkSuite {
     pub fn total_benchmark_count(&self) -> u64 {
         self.benchmark_names().count() as u64
     }
@@ -42,7 +42,7 @@ impl BenchmarkDatabase {
     }
 
     fn benchmark_names(&self) -> impl Iterator<Item = &str> {
-        self.suites
+        self.groups
             .iter()
             .flat_map(|suite| suite.benchmark_names.iter().map(|n| n.as_ref()))
     }
@@ -63,8 +63,8 @@ impl BenchmarkFilter {
 /// We assume that each binary defines a benchmark suite using `benchlib`.
 /// We then execute each benchmark suite with the `list-benchmarks` command to find out its
 /// benchmark names.
-pub fn discover_benchmarks(cargo_stdout: &[u8]) -> anyhow::Result<BenchmarkDatabase> {
-    let mut binaries = vec![];
+pub fn discover_benchmarks(cargo_stdout: &[u8]) -> anyhow::Result<BenchmarkSuite> {
+    let mut groups = vec![];
 
     for message in Message::parse_stream(cargo_stdout) {
         let message = message?;
@@ -79,7 +79,7 @@ pub fn discover_benchmarks(cargo_stdout: &[u8]) -> anyhow::Result<BenchmarkDatab
                                 path.display()
                             )
                         })?;
-                        binaries.push(BenchmarkSuite {
+                        groups.push(BenchmarkGroup {
                             binary: path,
                             benchmark_names: benchmarks,
                         });
@@ -90,10 +90,10 @@ pub fn discover_benchmarks(cargo_stdout: &[u8]) -> anyhow::Result<BenchmarkDatab
         }
     }
 
-    binaries.sort_unstable_by(|a, b| a.binary.cmp(&b.binary));
-    log::debug!("Found binaries: {:?}", binaries);
+    groups.sort_unstable_by(|a, b| a.binary.cmp(&b.binary));
+    log::debug!("Found binaries: {:?}", groups);
 
-    Ok(BenchmarkDatabase { suites: binaries })
+    Ok(BenchmarkSuite { groups })
 }
 
 /// Uses the `list-benchmarks` command from `benchlib` to find the benchmark names from the given

collector/src/runtime/mod.rs

@@ -12,28 +12,28 @@ pub use benchmark::BenchmarkFilter;
 /// Perform a series of runtime benchmarks using the provided `rustc` compiler.
 /// The runtime benchmarks are looked up in `benchmark_dir`, which is expected to be a path
 /// to a Cargo crate. All binaries built by that crate will are expected to be runtime benchmark
-/// suites that leverage `benchlib`.
+/// groups that leverage `benchlib`.
 pub fn bench_runtime(
     rustc: &str,
     id: Option<&str>,
     filter: BenchmarkFilter,
     benchmark_dir: PathBuf,
 ) -> anyhow::Result<()> {
     let toolchain = get_local_toolchain(&[Profile::Opt], rustc, None, None, id, "")?;
-    let output = compile_runtime_benchmarks(&toolchain, &benchmark_dir)?;
-    let benchmark_db = benchmark::discover_benchmarks(&output)?;
+    let output = compile_runtime_benchmark_binaries(&toolchain, &benchmark_dir)?;
+    let suite = benchmark::discover_benchmarks(&output)?;
 
-    let total_benchmark_count = benchmark_db.total_benchmark_count();
-    let filtered = benchmark_db.filtered_benchmark_count(&filter);
+    let total_benchmark_count = suite.total_benchmark_count();
+    let filtered = suite.filtered_benchmark_count(&filter);
     println!(
         "Executing {} benchmarks ({} filtered out)",
         filtered,
         total_benchmark_count - filtered
     );
 
     let mut benchmark_index = 0;
-    for binary in benchmark_db.suites {
-        for message in execute_runtime_benchmark(&binary.binary, &filter)? {
+    for binary in suite.groups {
+        for message in execute_runtime_benchmark_binary(&binary.binary, &filter)? {
             let message = message.map_err(|err| {
                 anyhow::anyhow!(
                     "Cannot parse BenchmarkMessage from benchmark {}: {err:?}",
@@ -59,10 +59,10 @@ pub fn bench_runtime(
     Ok(())
 }
 
-/// Starts executing a single runtime benchmark suite defined in a binary crate located in
-/// `runtime-benchmarks`. The binary is expected to use benchlib's `BenchmarkSuite` to execute
+/// Starts executing a single runtime benchmark group defined in a binary crate located in
+/// `runtime-benchmarks`. The binary is expected to use benchlib's `BenchmarkGroup` to execute
 /// a set of runtime benchmarks and print `BenchmarkMessage`s encoded as JSON, one per line.
-fn execute_runtime_benchmark(
+fn execute_runtime_benchmark_binary(
     binary: &Path,
     filter: &BenchmarkFilter,
 ) -> anyhow::Result<impl Iterator<Item = anyhow::Result<BenchmarkMessage>>> {
@@ -92,8 +92,11 @@ fn execute_runtime_benchmark(
     Ok(iterator)
 }
 
-/// Compiles all runtime benchmarks and returns the stdout output of Cargo.
-fn compile_runtime_benchmarks(toolchain: &LocalToolchain, dir: &Path) -> anyhow::Result<Vec<u8>> {
+/// Compiles all runtime benchmark binaries (groups) and returns the stdout output of Cargo.
+fn compile_runtime_benchmark_binaries(
+    toolchain: &LocalToolchain,
+    dir: &Path,
+) -> anyhow::Result<Vec<u8>> {
     let result = Command::new(&toolchain.cargo)
         .env("RUSTC", &toolchain.rustc)
         .arg("build")

docs/glossary.md

@@ -36,9 +36,6 @@ The following is a glossary of domain specific terminology. Although benchmarks
 * **relevant test result comparison**: a test result comparison can be significant but still not be relevant (i.e., worth paying attention to). Relevance is a factor of the test result comparison's significance and magnitude. Comparisons are considered relevant if they are significant and have at least a small magnitude .
 * **test result comparison magnitude**: how "large" the delta is between the two test result's under comparison. This is determined by the average of two factors: the absolute size of the change (i.e., a change of 5% is larger than a change of 1%) and the amount above the significance threshold (i.e., a change that is 5x the significance threshold is larger than a change 1.5x the significance threshold).
 
-## Runtime benchmarks
-* **benchmark suite**: a collection of individual benchmarks that is defined in a single binary.
-
 ## Other 
 
 * **bootstrap**: the process of building the compiler from a previous version of the compiler