diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0c990c9..955750a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,6 +1,17 @@ name: Benchmark on: + pull_request: + branches: + - main + paths: + - 'include/**' + - 'src/**' + - 'subprojects/**' + - 'meson.build' + - 'meson_options.txt' + - '.github/workflows/benchmark.yml' + - '*.cfg' workflow_dispatch: env: @@ -36,15 +47,15 @@ jobs: - name: Lint cpplint-cpp run: | - python benchmark.py . --cpplint_cpp="./build/cpplint-cpp" - sh memory_usage.sh "python cpplint.py" . - sh memory_usage.sh "./build/cpplint-cpp" . + python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp" + sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" . + sh ./benchmark/memory_usage.sh "python cpplint.py" . - name: Lint googletest run: | - python benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp" - sh memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }} - sh memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }} + python ./benchmark/benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp" + sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }} + sh ./benchmark/memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }} - name: Check required GLIBC version run: bash .github/workflows/check_glibc_compatibility.sh ./build/cpplint-cpp diff --git a/benchmark.py b/benchmark/benchmark.py similarity index 90% rename from benchmark.py rename to benchmark/benchmark.py index ed3b22c..0f48886 100644 --- a/benchmark.py +++ b/benchmark/benchmark.py @@ -1,9 +1,11 @@ +# Script to measure the average execution time for cpplint-cpp and cpplint.py. + import argparse import os import subprocess import time -def measure_time(command, repeat_time=30): +def measure_time(command, repeat_time=60): duration = 0 count = 0 while(duration < repeat_time): @@ -30,8 +32,8 @@ def get_args(): help="command to run cpplint.py") parser.add_argument("--options", default="--recursive --quiet --counting=detailed", type=str, help="options for cpplint") - parser.add_argument("--time", default=30, type=int, - help="Minimum measurement time for a command. Default to 30 (sec)") + parser.add_argument("--time", default=60, type=int, + help="Minimum measurement time for a command. Default to 60 (sec)") return parser.parse_args() diff --git a/memory_usage.sh b/benchmark/memory_usage.sh similarity index 61% rename from memory_usage.sh rename to benchmark/memory_usage.sh index 2f08fbe..0e3f3d8 100644 --- a/memory_usage.sh +++ b/benchmark/memory_usage.sh @@ -2,10 +2,10 @@ # Script to measure memory usage with cpplint # # Examples -# sh memory_usage.sh "python3 cpplint.py" . -# sh memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0 +# sh ./benchmark/memory_usage.sh "python3 cpplint.py" . +# sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0 -echo Measuring memory usage... +echo Measuring memory usage: $1 kib="$(/usr/bin/time -f "%M" sh -c "$1 --recursive --quiet --counting=detailed $2 >/dev/null 2>&1 || exit 0" 2>&1)" mib=$(echo "scale=2; $kib / 1024" | bc) echo Maximum memory usage: ${mib} MiB diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md new file mode 100644 index 0000000..0821ce7 --- /dev/null +++ b/docs/BENCHMARK.md @@ -0,0 +1,36 @@ +# Benchmarking + +This repository contains some scripts for benchmarking. + +## Execution time + +[`benchmark.py`](../benchmark/benchmark.py) can measure the average execution time for cpplint-cpp and cpplint.py. It takes about two minutes for measurement. + +```console +$ python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp" +Measuring time for cpplint-cpp: ./build/cpplint-cpp --recursive --quiet --counting=detailed . +Measuring time for cpplint.py: python cpplint.py --recursive --quiet --counting=detailed . +Execution time for cpplint-cpp: x.xxxxxx seconds +Execution time for cpplint.py: x.xxxxxx seconds +``` + +## Memory usage + +[`memory_usage.sh`](../benchmark/memory_usage.sh) can measure memory usage for a linter against a directory. + +```console +$ sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" . +Measuring memory usage: ./build/cpplint-cpp +Maximum memory usage: xx.xx MiB + +$ sh ./benchmark/memory_usage.sh "python cpplint.py" . +Measuring memory usage: python cpplint.py +Maximum memory usage: xx.xx MiB +``` + +## Github Actions + +You don't need to setup environment for benchmarking. +You can use [`benchmark.yml`](../.github/workflows/benchmark.yml) in your fork to run `benchmark.py` and `memory_usage.sh`. + +![benchmark.yml](https://github.com/user-attachments/assets/491f4c4b-32c4-4362-9406-2b1816e51574) diff --git a/docs/README.md b/docs/README.md index 5d6d271..c44fee2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,7 +21,7 @@ pip install cpplint-cpp --no-index --find-links https://matyalatte.github.io/cpp Here is an analysis of the performance differences between `cpplint-cpp` and `cpplint.py` against two repositories: [`googletest`](https://github.com/google/googletest) and `cpplint-cpp`. -Measurements were taken on an Ubuntu runner with [`benchmark.yml`](../.github/workflows/benchmark.yml). +Measurements were taken on an Ubuntu runner with [some scripts](BENCHMARK.md). ### Execution time