1
1
#! /usr/bin/env bash
2
2
3
3
if [ $# -lt 2 ]; then
4
- echo " usage: ./scripts/compare-commits.sh <commit1> <commit2> [additional llama-bench arguments]"
4
+ echo " usage: ./scripts/compare-commits.sh <commit1> <commit2> [tool] [additional arguments]"
5
+ echo " tool: 'llama-bench' (default) or 'test-backend-ops'"
6
+ echo " additional arguments: passed to the selected tool"
5
7
exit 1
6
8
fi
7
9
8
10
set -e
9
11
set -x
10
12
13
+ # Parse arguments
14
+ commit1=$1
15
+ commit2=$2
16
+ tool=${3:- llama-bench}
17
+ additional_args=" ${@: 4} "
18
+
19
+ # Validate tool argument
20
+ if [ " $tool " != " llama-bench" ] && [ " $tool " != " test-backend-ops" ]; then
21
+ echo " Error: tool must be 'llama-bench' or 'test-backend-ops'"
22
+ exit 1
23
+ fi
24
+
11
25
# verify at the start that the compare script has all the necessary dependencies installed
12
26
./scripts/compare-llama-bench.py --check
13
27
14
- bench_args=" ${@: 3} "
28
+ if [ " $tool " = " llama-bench" ]; then
29
+ db_file=" llama-bench.sqlite"
30
+ target=" llama-bench"
31
+ run_args=" -o sql -oe md $additional_args "
32
+ else # test-backend-ops
33
+ db_file=" test-backend-ops.sqlite"
34
+ target=" test-backend-ops"
35
+ run_args=" perf --output sql $additional_args "
36
+ fi
15
37
16
- rm -f llama-bench.sqlite > /dev/null
38
+ rm -f " $db_file " > /dev/null
17
39
18
40
# to test a backend, call the script with the corresponding environment variable (e.g. GGML_CUDA=1 ./scripts/compare-commits.sh ...)
19
41
if [ -n " $GGML_CUDA " ]; then
@@ -25,14 +47,14 @@ dir="build-bench"
25
47
function run {
26
48
rm -fr ${dir} > /dev/null
27
49
cmake -B ${dir} -S . ${CMAKE_OPTS} > /dev/null
28
- cmake --build ${dir} -t llama-bench > /dev/null
29
- ${dir} /bin/llama-bench -o sql -oe md $bench_args | sqlite3 llama-bench.sqlite
50
+ cmake --build ${dir} -t $target > /dev/null
51
+ ${dir} /bin/$target $run_args | sqlite3 " $db_file "
30
52
}
31
53
32
- git checkout $1 > /dev/null
54
+ git checkout $commit1 > /dev/null
33
55
run
34
56
35
- git checkout $2 > /dev/null
57
+ git checkout $commit2 > /dev/null
36
58
run
37
59
38
- ./scripts/compare-llama-bench.py -b $1 -c $2
60
+ ./scripts/compare-llama-bench.py -b $commit1 -c $commit2 --tool $tool -i " $db_file "
0 commit comments