Skip to content

Commit 305568e

Browse files
derekbitderekbit
committed
Expose P99 latency
Expose 99th percentile. It means that 99% of the requests should be faster than given latency. Longhorn 8254 Signed-off-by: Derek Su <derek.su@suse.com>
1 parent eaf910f commit 305568e

File tree

5 files changed

+108
-63
lines changed

5 files changed

+108
-63
lines changed

‎fio/cmp_parse.sh

Lines changed: 50 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ if [ -z "${2}" ]; then
1717
fi
1818
METRICS="${2}"
1919

20+
P99_LATENCY="${3:-false}"
21+
2022
if [ -z "FIRST_VOL_NAME" ]; then
2123
echo Require the first volume name
2224
exit 1
@@ -49,6 +51,9 @@ parse_metrics() {
4951
for IO_TYPE in "${io_types_array[@]}"; do
5052
local output="${vol_name}-${IO_TYPE}-${METRIC}.json"
5153
local parse_func="parse_${IO_TYPE}_${METRIC}"
54+
if [ "$P99_LATENCY" = "true" ]; then
55+
parse_func="parse_${IO_TYPE}_${METRIC}_p99"
56+
fi
5257

5358
if declare -f "$parse_func" > /dev/null; then
5459
$parse_func "$output"
@@ -80,8 +85,6 @@ calc_cmp_bandwidth
8085
calc_cmp_latency
8186

8287
# Build the summary with header information
83-
RESULT=${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}.summary
84-
8588
QUICK_MODE_TEXT="Quick Mode: disabled"
8689
if [ -n "$QUICK_MODE" ]; then
8790
QUICK_MODE_TEXT="Quick Mode: enabled"
@@ -92,22 +95,31 @@ if [ -n "$SIZE" ]; then
9295
SIZE_TEXT="Size: $SIZE"
9396
fi
9497

98+
# Determine the file name suffix and title based on P99_LATENCY
99+
if [ "$P99_LATENCY" = "false" ]; then
100+
FILE_SUFFIX="summary"
101+
TITLE="FIO Benchmark Comparison Summary"
102+
else
103+
FILE_SUFFIX="p99_latency.summary"
104+
TITLE="FIO Benchmark P99 Latency Comparison Summary"
105+
fi
106+
107+
RESULT="${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}_${FILE_SUFFIX}"
108+
95109
SUMMARY="
96-
================================
97-
FIO Benchmark Comparsion Summary
110+
===============================
111+
$TITLE
98112
For: $FIRST_VOL_NAME vs $SECOND_VOL_NAME
99113
CPU Idleness Profiling: $CPU_IDLE_PROF
100114
$SIZE_TEXT
101115
$QUICK_MODE_TEXT
102-
================================
116+
===============================
103117
"
104118

105119
printf -v header "$CMP_FMT" \
106120
"" $FIRST_VOL_NAME "vs" $SECOND_VOL_NAME ":" "Change"
107121
SUMMARY+=$header
108122

109-
#!/bin/bash
110-
111123
# Define a function to add metrics to the summary
112124
add_metrics_to_summary() {
113125
local metric_name="${1}"
@@ -139,7 +151,7 @@ add_metrics_to_summary() {
139151
local cmp_seqwrite="${25}"
140152
local cmp_cpu_idle_pct_seqwrite="${26}"
141153

142-
if [ "$CPU_IDLE_PROF" = "enabled" ]; then
154+
if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then
143155
printf -v cxt "${metric_name} in ${metric_unit} with CPU idleness in percent (Read/Write)\n${CMP_FMT}${CMP_FMT}${CMP_FMT}\n" \
144156
"Random:" \
145157
"$(commaize "${first_randread}") ($(commaize "${first_cpu_idle_pct_randread}")) / $(commaize "${first_randwrite}") ($(commaize "${first_cpu_idle_pct_randwrite}"))" \
@@ -174,33 +186,35 @@ if [ "x$CPU_IDLE_PROF" = "xenabled" ]; then
174186
fi
175187

176188
# Example usage
177-
add_metrics_to_summary "IOPS" "ops" \
178-
"$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \
179-
"$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \
180-
"$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \
181-
"$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \
182-
"$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \
183-
"$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \
184-
"$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \
185-
"$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \
186-
"$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \
187-
"$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \
188-
"$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \
189-
"$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS"
190-
191-
add_metrics_to_summary "Bandwidth" "KiB/sec" \
192-
"$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
193-
"$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
194-
"$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
195-
"$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
196-
"$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
197-
"$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
198-
"$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
199-
"$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
200-
"$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
201-
"$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
202-
"$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
203-
"$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
189+
if [ "$P99_LATENCY" = "false" ]; then
190+
add_metrics_to_summary "IOPS" "ops" \
191+
"$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \
192+
"$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \
193+
"$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \
194+
"$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \
195+
"$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \
196+
"$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \
197+
"$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \
198+
"$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \
199+
"$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \
200+
"$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \
201+
"$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \
202+
"$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS"
203+
204+
add_metrics_to_summary "Bandwidth" "KiB/sec" \
205+
"$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
206+
"$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
207+
"$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
208+
"$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
209+
"$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
210+
"$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
211+
"$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
212+
"$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
213+
"$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
214+
"$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
215+
"$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
216+
"$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
217+
fi
204218

205219
add_metrics_to_summary "Latency" "ns" \
206220
"$FIRST_RANDREAD_LATENCY" "$FIRST_CPU_IDLE_PCT_RANDREAD_LATENCY" \
@@ -218,6 +232,4 @@ add_metrics_to_summary "Latency" "ns" \
218232

219233

220234
echo "$SUMMARY" > $RESULT
221-
cat $RESULT
222-
223-
235+
cat $RESULT

‎fio/cmp_run.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@ if [ -z "SECOND_VOL_FILE" ]; then
2525
exit 1
2626
fi
2727

28-
#disable parsing in run.sh
29-
export SKIP_PARSE=1
30-
3128
$CURRENT_DIR/run.sh $FIRST_VOL_FILE $FIRST_VOL_NAME
3229
$CURRENT_DIR/run.sh $SECOND_VOL_FILE $SECOND_VOL_NAME
3330

34-
$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS"
31+
$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS" "false"
32+
$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "latency" "true"

‎fio/func.sh

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,29 @@ parse_randwrite_latency() {
7575
CPU_IDLE_PCT_RANDWRITE_LATENCY=`cat $OUTPUT | jq '.cpu_idleness.system' | cut -f1 -d.`
7676
}
7777

78+
# Latency 99th percentile
79+
parse_seqread_latency_p99() {
80+
local OUTPUT=${1}
81+
SEQREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
82+
}
83+
84+
parse_seqwrite_latency_p99() {
85+
local OUTPUT=${1}
86+
SEQWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
87+
}
88+
89+
parse_randread_latency_p99() {
90+
local OUTPUT=${1}
91+
RANDREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
92+
}
93+
94+
parse_randwrite_latency_p99() {
95+
local OUTPUT=${1}
96+
RANDWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
97+
}
98+
7899

79-
FMT="%25s%25s\n"
100+
FMT="%30s%30s\n"
80101
CMP_FMT="%20s%30s%10s%30s%10s%25s\n"
81102

82103
commaize() {

‎fio/parse.sh

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ append_metric() {
1717
local seqwrite="${9}"
1818
local cpu_idle_pct_seqwrite="${10}"
1919

20-
if [ "$CPU_IDLE_PROF" = "enabled" ]; then
20+
if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then
2121
# If CPU idle profiling is enabled, include it in the output
2222
printf -v cxt "%s in %s with CPU idleness in percent (Read/Write)\n$FMT$FMT\n" \
2323
"$metric_name" "$metric_unit" \
@@ -56,42 +56,57 @@ if [ -z "${3}" ]; then
5656
fi
5757
PREFIX="${3}"
5858

59+
P99_LATENCY="${4:-true}"
5960

6061
IFS=',' read -r -a io_types_array <<< "${IO_TYPES}"
6162
IFS=',' read -r -a metrics_array <<< "${METRICS}"
6263

6364
for TYPE in "${io_types_array[@]}"; do
6465
for METRIC in "${metrics_array[@]}"; do
6566
OUTPUT="${PREFIX}-${TYPE}-${METRIC}.json"
66-
parse_${TYPE}_${METRIC} "$OUTPUT"
67+
if [ "$P99_LATENCY" = "true" ]; then
68+
parse_${TYPE}_${METRIC}_p99 "$OUTPUT"
69+
else
70+
parse_${TYPE}_${METRIC} "$OUTPUT"
71+
fi
6772
done
6873
done
6974

7075
# Initialize the result file name
71-
RESULT=${PREFIX}.summary
72-
7376
# Build the summary with header information
77+
if [ "$P99_LATENCY" = "false" ]; then
78+
RESULT="${PREFIX}.summary"
79+
TITLE="FIO Benchmark Summary"
80+
else
81+
RESULT="${PREFIX}_p99_latency.summary"
82+
TITLE="FIO Benchmark P99 Latency Summary"
83+
fi
84+
85+
# Construct the SUMMARY with dynamic content
7486
SUMMARY="
75-
=========================
76-
FIO Benchmark Summary
87+
==================================
88+
$TITLE
7789
For: $PREFIX
7890
CPU Idleness Profiling: ${CPU_IDLE_PROF:-not provided}
7991
Size: ${SIZE:-10g}
8092
Quick Mode: ${QUICK_MODE:-disabled}
81-
=========================
93+
==================================
8294
"
8395

8496
# Append performance metrics to the summary
85-
append_metric "IOPS" "ops" \
86-
"$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \
87-
"$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \
88-
"$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \
89-
"$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS"
90-
append_metric "Bandwidth" "KiB/sec" \
91-
"$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
92-
"$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
93-
"$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
94-
"$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
97+
if [ "$P99_LATENCY" = "false" ]; then
98+
append_metric "IOPS" "ops" \
99+
"$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \
100+
"$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \
101+
"$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \
102+
"$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS"
103+
append_metric "Bandwidth" "KiB/sec" \
104+
"$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
105+
"$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
106+
"$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
107+
"$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
108+
fi
109+
95110
append_metric "Latency" "ns" \
96111
"$RANDREAD_LATENCY" "$CPU_IDLE_PCT_RANDREAD_LATENCY" \
97112
"$RANDWRITE_LATENCY" "$CPU_IDLE_PCT_RANDWRITE_LATENCY" \

‎fio/run.sh

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ for TYPE in "${io_types_array[@]}"; do
4242
done
4343
done
4444

45-
if [ -z "$SKIP_PARSE" ]; then
46-
"$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT"
47-
fi
45+
"$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT" "false"
46+
"$CURRENT_DIR/parse.sh" "$IO_TYPES" "latency" "$OUTPUT" "true"
4847

0 commit comments

Comments
 (0)