Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 697ce77

Browse files
committed
Add benchmarks using iai-callgrind
Running walltime benchmarks in CI is notoriously unstable, Introduce benchmarks that instead use instruction count and other more reproducible metrics, using `iai-callgrind` [1], which we are able to run in CI with a high degree of reproducibility. Inputs to this benchmark are a logspace sweep, which gives an approximation for real-world use, but may fail to indicate outlier cases. [1]: https://github.com/iai-callgrind/iai-callgrind
1 parent 342600b commit 697ce77

File tree

5 files changed

+192
-1
lines changed

5 files changed

+192
-1
lines changed

Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,7 @@ debug-assertions = true
7373
inherits = "release"
7474
lto = "fat"
7575
overflow-checks = true
76+
77+
[profile.bench]
78+
# Required for iai-callgrind
79+
debug = true

crates/libm-test/Cargo.toml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,17 @@ build-musl = ["dep:musl-math-sys"]
2020
# Enable report generation without bringing in more dependencies by default
2121
benchmarking-reports = ["criterion/plotters", "criterion/html_reports"]
2222

23+
# Enable icount benchmarks (requires iai-callgrind and valgrind)
24+
icount = ["dep:iai-callgrind"]
25+
2326
# Run with a reduced set of benchmarks, such as for CI
2427
short-benchmarks = []
2528

2629
[dependencies]
2730
anyhow = "1.0.90"
2831
az = { version = "1.2.1", optional = true }
2932
gmp-mpfr-sys = { version = "1.6.4", optional = true, default-features = false, features = ["mpfr"] }
33+
iai-callgrind = { version = "0.14.0", optional = true }
3034
indicatif = { version = "0.17.9", default-features = false }
3135
libm = { path = "../..", features = ["unstable-public-internals"] }
3236
libm-macros = { path = "../libm-macros" }
@@ -48,6 +52,11 @@ rand = { version = "0.8.5", optional = true }
4852
criterion = { version = "0.5.1", default-features = false, features = ["cargo_bench_support"] }
4953
libtest-mimic = "0.8.1"
5054

55+
[[bench]]
56+
name = "icount"
57+
harness = false
58+
required-features = ["icount"]
59+
5160
[[bench]]
5261
name = "random"
5362
harness = false

crates/libm-test/benches/icount.rs

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
//! Benchmarks that use `iai-cachegrind` to be reasonably CI-stable.
2+
3+
use std::hint::black_box;
4+
5+
use iai_callgrind::{library_benchmark, library_benchmark_group, main};
6+
use libm_test::gen::spaced;
7+
use libm_test::{CheckBasis, CheckCtx, GeneratorKind, MathOp, OpRustArgs, TupleCall, op};
8+
9+
const BENCH_ITER_ITEMS: u64 = 500;
10+
11+
macro_rules! icount_benches {
12+
(
13+
fn_name: $fn_name:ident,
14+
attrs: [$($_attr:meta),*],
15+
) => {
16+
paste::paste! {
17+
// Construct benchmark inputs from the logspace generator.
18+
fn [< setup_ $fn_name >]() -> Vec<OpRustArgs<op::$fn_name::Routine>> {
19+
type Op = op::$fn_name::Routine;
20+
let mut ctx = CheckCtx::new(
21+
Op::IDENTIFIER,
22+
CheckBasis::None,
23+
GeneratorKind::QuickSpaced
24+
);
25+
ctx.override_iterations(BENCH_ITER_ITEMS);
26+
let ret = spaced::get_test_cases::<Op>(&ctx).0.collect::<Vec<_>>();
27+
println!("operation {}, {} steps", Op::NAME, ret.len());
28+
ret
29+
}
30+
31+
// Run benchmarks with the above inputs.
32+
#[library_benchmark]
33+
#[bench::logspace([< setup_ $fn_name >]())]
34+
fn [< icount_bench_ $fn_name >](cases: Vec<OpRustArgs<op::$fn_name::Routine>>) {
35+
type Op = op::$fn_name::Routine;
36+
let f = black_box(Op::ROUTINE);
37+
for input in cases.iter().copied() {
38+
input.call(f);
39+
}
40+
}
41+
42+
library_benchmark_group!(
43+
name = [< icount_bench_ $fn_name _group >];
44+
benchmarks = [< icount_bench_ $fn_name >]
45+
);
46+
}
47+
};
48+
}
49+
50+
libm_macros::for_each_function! {
51+
callback: icount_benches,
52+
}
53+
54+
main!(
55+
library_benchmark_groups = icount_bench_acos_group,
56+
icount_bench_acosf_group,
57+
icount_bench_acosh_group,
58+
icount_bench_acoshf_group,
59+
icount_bench_asin_group,
60+
icount_bench_asinf_group,
61+
icount_bench_asinh_group,
62+
icount_bench_asinhf_group,
63+
icount_bench_atan2_group,
64+
icount_bench_atan2f_group,
65+
icount_bench_atan_group,
66+
icount_bench_atanf_group,
67+
icount_bench_atanh_group,
68+
icount_bench_atanhf_group,
69+
icount_bench_cbrt_group,
70+
icount_bench_cbrtf_group,
71+
icount_bench_ceil_group,
72+
icount_bench_ceilf_group,
73+
icount_bench_copysign_group,
74+
icount_bench_copysignf128_group,
75+
icount_bench_copysignf16_group,
76+
icount_bench_copysignf_group,
77+
icount_bench_cos_group,
78+
icount_bench_cosf_group,
79+
icount_bench_cosh_group,
80+
icount_bench_coshf_group,
81+
icount_bench_erf_group,
82+
icount_bench_erfc_group,
83+
icount_bench_erfcf_group,
84+
icount_bench_erff_group,
85+
icount_bench_exp10_group,
86+
icount_bench_exp10f_group,
87+
icount_bench_exp2_group,
88+
icount_bench_exp2f_group,
89+
icount_bench_exp_group,
90+
icount_bench_expf_group,
91+
icount_bench_expm1_group,
92+
icount_bench_expm1f_group,
93+
icount_bench_fabs_group,
94+
icount_bench_fabsf128_group,
95+
icount_bench_fabsf16_group,
96+
icount_bench_fabsf_group,
97+
icount_bench_fdim_group,
98+
icount_bench_fdimf128_group,
99+
icount_bench_fdimf16_group,
100+
icount_bench_fdimf_group,
101+
icount_bench_floor_group,
102+
icount_bench_floorf_group,
103+
icount_bench_fma_group,
104+
icount_bench_fmaf_group,
105+
icount_bench_fmax_group,
106+
icount_bench_fmaxf_group,
107+
icount_bench_fmin_group,
108+
icount_bench_fminf_group,
109+
icount_bench_fmod_group,
110+
icount_bench_fmodf_group,
111+
icount_bench_frexp_group,
112+
icount_bench_frexpf_group,
113+
icount_bench_hypot_group,
114+
icount_bench_hypotf_group,
115+
icount_bench_ilogb_group,
116+
icount_bench_ilogbf_group,
117+
icount_bench_j0_group,
118+
icount_bench_j0f_group,
119+
icount_bench_j1_group,
120+
icount_bench_j1f_group,
121+
icount_bench_jn_group,
122+
icount_bench_jnf_group,
123+
icount_bench_ldexp_group,
124+
icount_bench_ldexpf_group,
125+
icount_bench_lgamma_group,
126+
icount_bench_lgamma_r_group,
127+
icount_bench_lgammaf_group,
128+
icount_bench_lgammaf_r_group,
129+
icount_bench_log10_group,
130+
icount_bench_log10f_group,
131+
icount_bench_log1p_group,
132+
icount_bench_log1pf_group,
133+
icount_bench_log2_group,
134+
icount_bench_log2f_group,
135+
icount_bench_log_group,
136+
icount_bench_logf_group,
137+
icount_bench_modf_group,
138+
icount_bench_modff_group,
139+
icount_bench_nextafter_group,
140+
icount_bench_nextafterf_group,
141+
icount_bench_pow_group,
142+
icount_bench_powf_group,
143+
icount_bench_remainder_group,
144+
icount_bench_remainderf_group,
145+
icount_bench_remquo_group,
146+
icount_bench_remquof_group,
147+
icount_bench_rint_group,
148+
icount_bench_rintf_group,
149+
icount_bench_round_group,
150+
icount_bench_roundf_group,
151+
icount_bench_scalbn_group,
152+
icount_bench_scalbnf_group,
153+
icount_bench_sin_group,
154+
icount_bench_sinf_group,
155+
icount_bench_sinh_group,
156+
icount_bench_sinhf_group,
157+
icount_bench_sqrt_group,
158+
icount_bench_sqrtf_group,
159+
icount_bench_tan_group,
160+
icount_bench_tanf_group,
161+
icount_bench_tanh_group,
162+
icount_bench_tanhf_group,
163+
icount_bench_tgamma_group,
164+
icount_bench_tgammaf_group,
165+
icount_bench_trunc_group,
166+
icount_bench_truncf128_group,
167+
icount_bench_truncf16_group,
168+
icount_bench_truncf_group,
169+
icount_bench_y0_group,
170+
icount_bench_y0f_group,
171+
icount_bench_y1_group,
172+
icount_bench_y1f_group,
173+
icount_bench_yn_group,
174+
icount_bench_ynf_group,
175+
);

crates/libm-test/src/lib.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ pub use f8_impl::f8;
2424
pub use libm::support::{Float, Int, IntTy, MinInt};
2525
pub use num::{FloatExt, linear_ints, logspace};
2626
pub use op::{
27-
BaseName, FloatTy, Identifier, MathOp, OpCFn, OpCRet, OpFTy, OpRustFn, OpRustRet, Ty,
27+
BaseName, FloatTy, Identifier, MathOp, OpCFn, OpCRet, OpFTy, OpRustArgs, OpRustFn, OpRustRet,
28+
Ty,
2829
};
2930
pub use precision::{MaybeOverride, SpecialCase, default_ulp};
3031
use run_cfg::EXTENSIVE_MAX_ITERATIONS;

crates/libm-test/src/op.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ pub type OpCFn<Op> = <Op as MathOp>::CFn;
100100
pub type OpCRet<Op> = <Op as MathOp>::CRet;
101101
/// Access the associated `RustFn` type from an op (helper to avoid ambiguous associated types).
102102
pub type OpRustFn<Op> = <Op as MathOp>::RustFn;
103+
/// Access the associated `RustArgs` type from an op (helper to avoid ambiguous associated types).
104+
pub type OpRustArgs<Op> = <Op as MathOp>::RustArgs;
103105
/// Access the associated `RustRet` type from an op (helper to avoid ambiguous associated types).
104106
pub type OpRustRet<Op> = <Op as MathOp>::RustRet;
105107

0 commit comments

Comments
 (0)