Skip to content

Commit 63dfc14

Browse files
shawntabriziarkparkianenigma
authored
Add DB Read/Write Tracking to Benchmarking Pipeline (#6386)
* initial mockup * add and wipe * track writes * start to add to pipeline * return all reads/writes * Log reads and writes from bench db * causes panic * Allow multiple commits * commit before ending benchmark * doesn't work??? * fix * Update lib.rs * switch to struct for `BenchmarkResults` * add to output * fix test * line width * @kianenigma review * Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405) * hardcoded whitelist * Add whitelist to pipeline * Remove whitelist pipeline from CLI, add to runtime * clean-up unused db initialized whitelist * Add regression analysis to DB Tracking (#6475) * Add selector * add tests * debug formatter for easy formula * Update client/db/src/bench.rs Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: arkpar <arkady.paronyan@gmail.com> Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
1 parent ac6e42b commit 63dfc14

File tree

3 files changed

+180
-42
lines changed

3 files changed

+180
-42
lines changed

‎src/analysis.rs

Lines changed: 105 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -29,24 +29,40 @@ pub struct Analysis {
2929
model: Option<RegressionModel>,
3030
}
3131

32+
pub enum BenchmarkSelector {
33+
ExtrinsicTime,
34+
StorageRootTime,
35+
Reads,
36+
Writes,
37+
}
38+
3239
impl Analysis {
33-
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
34-
let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| {
40+
pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
41+
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
3542
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
36-
for (params, _, _) in r.iter() {
37-
let mut p = params.iter().map(|x| x.1).collect::<Vec<_>>();
43+
for result in r.iter() {
44+
let mut p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
3845
p[i] = 0;
3946
*counted.entry(p).or_default() += 1;
4047
}
4148
let others: Vec<u32> = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone();
4249
let values = r.iter()
4350
.filter(|v|
44-
v.0.iter()
51+
v.components.iter()
4552
.map(|x| x.1)
4653
.zip(others.iter())
4754
.enumerate()
4855
.all(|(j, (v1, v2))| j == i || v1 == *v2)
49-
).map(|(ps, v, _)| (ps[i].1, *v))
56+
).map(|result| {
57+
// Extract the data we are interested in analyzing
58+
let data = match selector {
59+
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
60+
BenchmarkSelector::StorageRootTime => result.storage_root_time,
61+
BenchmarkSelector::Reads => result.reads.into(),
62+
BenchmarkSelector::Writes => result.writes.into(),
63+
};
64+
(result.components[i].1, data)
65+
})
5066
.collect::<Vec<_>>();
5167
(format!("{:?}", param), i, others, values)
5268
}).collect::<Vec<_>>();
@@ -97,12 +113,18 @@ impl Analysis {
97113
})
98114
}
99115

100-
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
116+
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
101117
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
102-
for &(ref params, t, _) in r.iter() {
103-
let p = params.iter().map(|x| x.1).collect::<Vec<_>>();
104-
results.entry(p).or_default().push(t);
118+
for result in r.iter() {
119+
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
120+
results.entry(p).or_default().push(match selector {
121+
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
122+
BenchmarkSelector::StorageRootTime => result.storage_root_time,
123+
BenchmarkSelector::Reads => result.reads.into(),
124+
BenchmarkSelector::Writes => result.writes.into(),
125+
})
105126
}
127+
106128
for (_, rs) in results.iter_mut() {
107129
rs.sort();
108130
let ql = rs.len() / 4;
@@ -111,7 +133,7 @@ impl Analysis {
111133

112134
let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())];
113135

114-
let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
136+
let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
115137
data.extend(names.iter()
116138
.enumerate()
117139
.map(|(i, p)| (
@@ -217,40 +239,88 @@ impl std::fmt::Display for Analysis {
217239
}
218240
}
219241

242+
impl std::fmt::Debug for Analysis {
243+
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
244+
write!(f, "{}", self.base)?;
245+
for (&m, n) in self.slopes.iter().zip(self.names.iter()) {
246+
write!(f, " + ({} * {})", m, n)?;
247+
}
248+
write!(f,"")
249+
}
250+
}
251+
220252
#[cfg(test)]
221253
mod tests {
222254
use super::*;
223255
use crate::BenchmarkParameter;
224256

257+
fn benchmark_result(
258+
components: Vec<(BenchmarkParameter, u32)>,
259+
extrinsic_time: u128,
260+
storage_root_time: u128,
261+
reads: u32,
262+
writes: u32,
263+
) -> BenchmarkResults {
264+
BenchmarkResults {
265+
components,
266+
extrinsic_time,
267+
storage_root_time,
268+
reads,
269+
repeat_reads: 0,
270+
writes,
271+
repeat_writes: 0,
272+
}
273+
}
274+
225275
#[test]
226276
fn analysis_median_slopes_should_work() {
227-
let a = Analysis::median_slopes(&vec![
228-
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
229-
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
230-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
231-
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
232-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
233-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
234-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
235-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
236-
]).unwrap();
237-
assert_eq!(a.base, 10_000_000);
238-
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
277+
let data = vec![
278+
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
279+
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
280+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
281+
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
282+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
283+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
284+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
285+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
286+
];
287+
288+
let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
289+
assert_eq!(extrinsic_time.base, 10_000_000);
290+
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
291+
292+
let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap();
293+
assert_eq!(reads.base, 2);
294+
assert_eq!(reads.slopes, vec![1, 0]);
295+
296+
let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap();
297+
assert_eq!(writes.base, 0);
298+
assert_eq!(writes.slopes, vec![0, 2]);
239299
}
240300

241301
#[test]
242302
fn analysis_median_min_squares_should_work() {
243-
let a = Analysis::min_squares_iqr(&vec![
244-
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
245-
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
246-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
247-
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
248-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
249-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
250-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
251-
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
252-
]).unwrap();
253-
assert_eq!(a.base, 10_000_000);
254-
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
303+
let data = vec![
304+
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
305+
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
306+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
307+
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
308+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
309+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
310+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
311+
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
312+
];
313+
314+
let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
315+
assert_eq!(extrinsic_time.base, 10_000_000);
316+
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
317+
318+
let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap();
319+
assert_eq!(reads.base, 2);
320+
assert_eq!(reads.slopes, vec![1, 0]);
321+
322+
let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap();
323+
assert_eq!(writes.base, 0);
324+
assert_eq!(writes.slopes, vec![0, 2]);
255325
}
256326
}

‎src/lib.rs

Lines changed: 50 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ mod analysis;
2626

2727
pub use utils::*;
2828
#[cfg(feature = "std")]
29-
pub use analysis::Analysis;
29+
pub use analysis::{Analysis, BenchmarkSelector};
3030
#[doc(hidden)]
3131
pub use sp_io::storage::root as storage_root;
3232
pub use sp_runtime::traits::Zero;
@@ -771,6 +771,7 @@ macro_rules! impl_benchmark {
771771
highest_range_values: &[u32],
772772
steps: &[u32],
773773
repeat: u32,
774+
whitelist: &[Vec<u8>]
774775
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
775776
// Map the input to the selected benchmark.
776777
let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -780,6 +781,9 @@ macro_rules! impl_benchmark {
780781
_ => return Err("Could not find extrinsic."),
781782
};
782783

784+
// Add whitelist to DB
785+
$crate::benchmarking::set_whitelist(whitelist.to_vec());
786+
783787
// Warm up the DB
784788
$crate::benchmarking::commit_db();
785789
$crate::benchmarking::wipe_db();
@@ -841,6 +845,9 @@ macro_rules! impl_benchmark {
841845
// This will enable worst case scenario for reading from the database.
842846
$crate::benchmarking::commit_db();
843847

848+
// Reset the read/write counter so we don't count operations in the setup process.
849+
$crate::benchmarking::reset_read_write_count();
850+
844851
// Time the extrinsic logic.
845852
frame_support::debug::trace!(
846853
target: "benchmark",
@@ -851,19 +858,33 @@ macro_rules! impl_benchmark {
851858
closure_to_benchmark()?;
852859
let finish_extrinsic = $crate::benchmarking::current_time();
853860
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
854-
861+
// Commit the changes to get proper write count
862+
$crate::benchmarking::commit_db();
855863
frame_support::debug::trace!(
856864
target: "benchmark",
857865
"End Benchmark: {} ns", elapsed_extrinsic
858866
);
867+
let read_write_count = $crate::benchmarking::read_write_count();
868+
frame_support::debug::trace!(
869+
target: "benchmark",
870+
"Read/Write Count {:?}", read_write_count
871+
);
859872

860873
// Time the storage root recalculation.
861874
let start_storage_root = $crate::benchmarking::current_time();
862875
$crate::storage_root();
863876
let finish_storage_root = $crate::benchmarking::current_time();
864877
let elapsed_storage_root = finish_storage_root - start_storage_root;
865878

866-
results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root));
879+
results.push($crate::BenchmarkResults {
880+
components: c.clone(),
881+
extrinsic_time: elapsed_extrinsic,
882+
storage_root_time: elapsed_storage_root,
883+
reads: read_write_count.0,
884+
repeat_reads: read_write_count.1,
885+
writes: read_write_count.2,
886+
repeat_writes: read_write_count.3,
887+
});
867888

868889
// Wipe the DB back to the genesis state.
869890
$crate::benchmarking::wipe_db();
@@ -892,6 +913,7 @@ macro_rules! impl_benchmark {
892913
highest_range_values: &[u32],
893914
steps: &[u32],
894915
repeat: u32,
916+
whitelist: &[Vec<u8>]
895917
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
896918
// Map the input to the selected benchmark.
897919
let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -901,6 +923,9 @@ macro_rules! impl_benchmark {
901923
_ => return Err("Could not find extrinsic."),
902924
};
903925

926+
// Add whitelist to DB
927+
$crate::benchmarking::set_whitelist(whitelist.to_vec());
928+
904929
// Warm up the DB
905930
$crate::benchmarking::commit_db();
906931
$crate::benchmarking::wipe_db();
@@ -963,6 +988,9 @@ macro_rules! impl_benchmark {
963988
// This will enable worst case scenario for reading from the database.
964989
$crate::benchmarking::commit_db();
965990

991+
// Reset the read/write counter so we don't count operations in the setup process.
992+
$crate::benchmarking::reset_read_write_count();
993+
966994
// Time the extrinsic logic.
967995
frame_support::debug::trace!(
968996
target: "benchmark",
@@ -973,19 +1001,33 @@ macro_rules! impl_benchmark {
9731001
closure_to_benchmark()?;
9741002
let finish_extrinsic = $crate::benchmarking::current_time();
9751003
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
976-
1004+
// Commit the changes to get proper write count
1005+
$crate::benchmarking::commit_db();
9771006
frame_support::debug::trace!(
9781007
target: "benchmark",
9791008
"End Benchmark: {} ns", elapsed_extrinsic
9801009
);
1010+
let read_write_count = $crate::benchmarking::read_write_count();
1011+
frame_support::debug::trace!(
1012+
target: "benchmark",
1013+
"Read/Write Count {:?}", read_write_count
1014+
);
9811015

9821016
// Time the storage root recalculation.
9831017
let start_storage_root = $crate::benchmarking::current_time();
9841018
$crate::storage_root();
9851019
let finish_storage_root = $crate::benchmarking::current_time();
9861020
let elapsed_storage_root = finish_storage_root - start_storage_root;
9871021

988-
results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root));
1022+
results.push($crate::BenchmarkResults {
1023+
components: c.clone(),
1024+
extrinsic_time: elapsed_extrinsic,
1025+
storage_root_time: elapsed_storage_root,
1026+
reads: read_write_count.0,
1027+
repeat_reads: read_write_count.1,
1028+
writes: read_write_count.2,
1029+
repeat_writes: read_write_count.3,
1030+
});
9891031

9901032
// Wipe the DB back to the genesis state.
9911033
$crate::benchmarking::wipe_db();
@@ -1139,7 +1181,7 @@ macro_rules! impl_benchmark_test {
11391181
#[macro_export]
11401182
macro_rules! add_benchmark {
11411183
( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
1142-
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params;
1184+
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params;
11431185
if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
11441186
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
11451187
for benchmark in $( $location )*::benchmarks().into_iter() {
@@ -1150,6 +1192,7 @@ macro_rules! add_benchmark {
11501192
&highest_range_values[..],
11511193
&steps[..],
11521194
repeat,
1195+
whitelist,
11531196
)?,
11541197
pallet: $name.to_vec(),
11551198
benchmark: benchmark.to_vec(),
@@ -1163,6 +1206,7 @@ macro_rules! add_benchmark {
11631206
&highest_range_values[..],
11641207
&steps[..],
11651208
repeat,
1209+
whitelist,
11661210
)?,
11671211
pallet: $name.to_vec(),
11681212
benchmark: benchmark.clone(),

0 commit comments

Comments
 (0)