Merge pull request #1644 from Kobzol/runtime-benchmark-harness

Make runtime benchmark harness more flexible to use

Author: Kobzol

collector/benchlib/src/benchmark.rs

@@ -7,30 +7,26 @@ use std::collections::HashMap;
 
 /// Create and run a new benchmark group. Use the closure argument to register
 /// the individual benchmarks.
-pub fn run_benchmark_group<F>(register: F)
+pub fn run_benchmark_group<'a, F>(register: F)
 where
-    F: FnOnce(&mut BenchmarkGroup),
+    F: FnOnce(&mut BenchmarkGroup<'a>),
 {
     env_logger::init();
 
-    let mut group = BenchmarkGroup::new();
+    let mut group: BenchmarkGroup<'a> = BenchmarkGroup::default();
     register(&mut group);
     group.run().expect("Benchmark group execution has failed");
 }
 
 /// Type-erased function that executes a single benchmark.
-type BenchmarkFn = Box<dyn Fn() -> anyhow::Result<BenchmarkStats>>;
+type BenchmarkFn<'a> = Box<dyn Fn() -> anyhow::Result<BenchmarkStats> + 'a>;
 
 #[derive(Default)]
-pub struct BenchmarkGroup {
-    benchmarks: HashMap<&'static str, BenchmarkFn>,
+pub struct BenchmarkGroup<'a> {
+    benchmarks: HashMap<&'static str, BenchmarkFn<'a>>,
 }
 
-impl BenchmarkGroup {
-    pub fn new() -> Self {
-        Self::default()
-    }
-
+impl<'a> BenchmarkGroup<'a> {
     /// Registers a single benchmark.
     ///
     /// `constructor` returns a closure that will be benchmarked. This means
@@ -40,19 +36,19 @@ impl BenchmarkGroup {
     /// closure it produces each time will only be called once.
     pub fn register_benchmark<Ctor, Bench, R>(&mut self, name: &'static str, constructor: Ctor)
     where
-        Ctor: Fn() -> Bench + Clone + 'static,
-        Bench: FnOnce() -> R + 'static,
+        Ctor: Fn() -> Bench + 'a,
+        Bench: FnOnce() -> R,
     {
         // We want to type-erase the target `func` by wrapping it in a Box.
-        let benchmark_fn = Box::new(move || benchmark_function(constructor.clone()));
+        let benchmark_fn = Box::new(move || benchmark_function(&constructor));
         if self.benchmarks.insert(name, benchmark_fn).is_some() {
             panic!("Benchmark '{}' was registered twice", name);
         }
     }
 
     /// Execute the benchmark group. It will parse CLI arguments and decide what to do based on
     /// them.
-    pub fn run(self) -> anyhow::Result<()> {
+    fn run(self) -> anyhow::Result<()> {
         raise_process_priority();
 
         let args = parse_cli()?;

collector/benchlib/src/measure/perf_counter/linux.rs

@@ -17,8 +17,8 @@ struct Counters {
 /// Benchmarks a single function generated by `benchmark_constructor`.
 /// The function is executed twice, once to gather wall-time measurement and the second time to
 /// gather perf. counters.
-pub fn benchmark_function<F: Fn() -> Bench + 'static, R, Bench: FnOnce() -> R + 'static>(
-    benchmark_constructor: F,
+pub fn benchmark_function<F: Fn() -> Bench, R, Bench: FnOnce() -> R>(
+    benchmark_constructor: &F,
 ) -> anyhow::Result<BenchmarkStats> {
     let mut group = create_group()?;
     let counters = prepare_counters(&mut group)?;

collector/runtime-benchmarks/README.md

@@ -2,8 +2,23 @@
 This directory contains various pieces of code for which we measure how fast do they execute
 when they are compiled with a specific version of `rustc`.
 
-The benchmarks are located in crates that are part of the `runtime-benchmarks` workspace. Each crate
-contains a set of benchmarks defined using named closures.
+The benchmarks are located in several crates (`benchmark groups`) located in this directory. Each
+group defines a set of benchmarks using named closures. Each group should have a single,
+default, binary target, which will be executed by `collector`, and it should use the
+`run_benchmark_group` function from [`benchlib`](../benchlib) to define the benchmarks.
 
-Benchmarks are divided into sub-crates so that some benchmarks can use various versions of dependency
-crates and also so that they are grouped together by a relevant area (e.g. hashmap benchmarks).
+Runtime benchmarks are divided into groups so that some benchmarks can use different versions of
+dependency crates and also so that they are grouped together by a relevant area
+(e.g. hashmap benchmarks).
+
+## How are benchmarks executed
+The `collector` compiles each benchmark group and then invokes it with the `list` argument to list
+all benchmarks contained in the group.
+
+Then it executes each group with the `run` argument, which will cause `benchlib` to actually perform
+the benchmarks and output the results to `stdout` in JSON. That means that the benchmarked code should
+not print anything to `stdout`!
+
+Note that each benchmark group binary will be thus invoked twice per benchmarking collection. Keep this
+in mind so that the `main` function of the group doesn't perform too much work, which will be thrown
+away when it is invoked with the `list` argument.

collector/runtime-benchmarks/bufreader/src/main.rs

@@ -5,16 +5,17 @@ use std::io::{BufRead, BufReader, Write};
 const BYTES: usize = 64 * 1024 * 1024;
 
 fn main() {
+    let data = vec![0u8; BYTES];
+
     // Inspired by https://github.com/rust-lang/rust/issues/102727
     // The pattern we want is a BufReader which wraps a Read impl where one Read::read call will
     // never fill the whole BufReader buffer.
     run_benchmark_group(|group| {
         group.register_benchmark("bufreader_snappy", || {
-            let data = vec![0u8; BYTES];
-            move || {
+            || {
                 let mut compressed = Vec::new();
                 FrameEncoder::new(&mut compressed)
-                    .write_all(&data[..])
+                    .write_all(data.as_slice())
                     .unwrap();
                 let mut reader =
                     BufReader::with_capacity(BYTES, FrameDecoder::new(&compressed[..]));
@@ -27,6 +28,7 @@ fn main() {
                     let len = buf.len();
                     reader.consume(len);
                 }
+                compressed
             }
         });
     });

collector/runtime-benchmarks/hashmap/src/main.rs

@@ -1,15 +1,29 @@
+use fxhash::FxBuildHasher;
+use hashbrown::HashMap;
+
 use benchlib;
 use benchlib::benchmark::{black_box, run_benchmark_group};
 
+fn create_map_1m_integers() -> HashMap<u64, u64, FxBuildHasher> {
+    let mut map: HashMap<u64, u64, _> =
+        HashMap::with_capacity_and_hasher(1_000_000, FxBuildHasher::default());
+    for index in 0..map.capacity() {
+        map.insert(index as u64, index as u64);
+    }
+    map
+}
+
 fn main() {
+    let map_1m_integers = create_map_1m_integers();
+
     run_benchmark_group(|group| {
         // Measures how long does it take to insert 1 million numbers into a hashmap.
         group.register_benchmark("hashmap_insert_1m", || {
             let count = 1_000_000;
-            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
+            let mut map = HashMap::with_capacity_and_hasher(
                 // Over allocate the hashmap to avoid reallocations when inserting
                 count * 2,
-                fxhash::FxBuildHasher::default(),
+                FxBuildHasher::default(),
             );
             move || {
                 for index in 0..count {
@@ -20,66 +34,37 @@ fn main() {
 
         // Measures how long it takes to remove 1 million elements from a hashmap.
         group.register_benchmark("hashmap_remove_1m", || {
-            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
-                1_000_000,
-                fxhash::FxBuildHasher::default(),
-            );
-            for index in 0..map.capacity() {
-                map.insert(index, index);
-            }
-
+            let mut map = create_map_1m_integers();
             move || {
                 for index in 0..map.capacity() {
-                    map.remove(&index);
+                    map.remove(&(index as u64));
                 }
             }
         });
 
         // Measures how long it takes to find 1 million elements that are in a hashmap.
         group.register_benchmark("hashmap_find_1m", || {
-            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
-                1_000_000,
-                fxhash::FxBuildHasher::default(),
-            );
-            for index in 0..map.capacity() {
-                map.insert(index, index);
-            }
-
-            move || {
+            || {
+                let map = &map_1m_integers;
                 for index in 0..map.capacity() {
-                    black_box(map.get(&index));
+                    black_box(map.get(&(index as u64)));
                 }
             }
         });
 
         // Measures how long it takes to find 1 million elements that are not in a hashmap.
         group.register_benchmark("hashmap_find_misses_1m", || {
-            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
-                1_000_000,
-                fxhash::FxBuildHasher::default(),
-            );
-            for index in 0..map.capacity() {
-                map.insert(index, index);
-            }
-
-            move || {
+            || {
+                let map = &map_1m_integers;
                 for index in map.capacity()..(map.capacity() * 2) {
-                    black_box(map.get(&index));
+                    black_box(map.get(&(index as u64)));
                 }
             }
         });
 
         // Measures how long it takes to iterate through values of a hashmap with 1 million elements.
         group.register_benchmark("hashmap_iterate_1m", || {
-            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
-                1_000_000,
-                fxhash::FxBuildHasher::default(),
-            );
-            for index in 0..map.capacity() {
-                map.insert(index, index as u64);
-            }
-
-            move || map.values().sum::<u64>()
+            || map_1m_integers.values().sum::<u64>()
         });
     });
 }

collector/runtime-benchmarks/raytracer/src/main.rs

@@ -2,7 +2,6 @@ use benchlib::benchmark::run_benchmark_group;
 use camera::Camera;
 use materials::{Dielectric, Lambertian, Material, Metal};
 use model::{Model, Sphere};
-use std::rc::Rc;
 use vec::{random_in_unit_disc, Vec3};
 
 mod camera; // translate 2D pixel coordinates to 3D rays
@@ -78,26 +77,27 @@ fn main() {
 
     const NSAMPLES: usize = 100;
 
+    let scene = create_scene();
+    let lookfrom = Vec3(20.0 * 0.47f32.cos(), 20.0 * 0.47f32.sin(), 3.0);
+    let lookat = Vec3(0.0, 0.0, 1.0);
+    let vup = Vec3(0.0, 0.0, 1.0);
+    let focus_distance = (lookfrom - lookat).length();
+    let aperture = 0.3;
+    let camera = Camera::new(
+        lookfrom,
+        lookat,
+        vup,
+        20.0,
+        WIDTH as f32 / HEIGHT as f32,
+        aperture,
+        focus_distance,
+    );
+
     run_benchmark_group(|group| {
         // Performs raytracing on a simple scene.
         // Adapted from https://github.com/jorendorff/rust-raytrace.
         group.register_benchmark("raytracer", || {
-            let scene = create_scene();
-            let lookfrom = Vec3(20.0 * 0.47f32.cos(), 20.0 * 0.47f32.sin(), 3.0);
-            let lookat = Vec3(0.0, 0.0, 1.0);
-            let vup = Vec3(0.0, 0.0, 1.0);
-            let focus_distance = (lookfrom - lookat).length();
-            let aperture = 0.3;
-            let camera = Rc::new(Camera::new(
-                lookfrom,
-                lookat,
-                vup,
-                20.0,
-                WIDTH as f32 / HEIGHT as f32,
-                aperture,
-                focus_distance,
-            ));
-            move || render::render(&*scene, &camera, WIDTH, HEIGHT, NSAMPLES)
+            || render::render(scene.as_ref(), &camera, WIDTH, HEIGHT, NSAMPLES)
         });
     });
 }

collector/runtime-benchmarks/text-search/src/main.rs

@@ -5,14 +5,15 @@ use benchlib::benchmark::run_benchmark_group;
 const TEXT_SHERLOCK: &str = include_str!("data/sherlock.txt");
 
 fn main() {
+    let regex1 = Regex::new(r"[a-zA-Z]+ing").unwrap();
+    let regex2 = Regex::new(r"(Sherlock|Holmes|Watson|Irene|Adler|John|Baker)").unwrap();
+
     run_benchmark_group(|group| {
         group.register_benchmark("regex-search-1", || {
-            let regex = Regex::new(r"[a-zA-Z]+ing").unwrap();
-            move || regex.find_iter(TEXT_SHERLOCK).count()
+            || regex1.find_iter(TEXT_SHERLOCK).count()
         });
         group.register_benchmark("regex-capture-1", || {
-            let regex = Regex::new(r"(Sherlock|Holmes|Watson|Irene|Adler|John|Baker)").unwrap();
-            move || regex.captures_iter(TEXT_SHERLOCK).count()
+            || regex2.captures_iter(TEXT_SHERLOCK).count()
         });
     });
 }