Improve benchmark.yml

.github/workflows/benchmark.yml

@@ -1,6 +1,17 @@
 name: Benchmark
 
 on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'include/**'
+      - 'src/**'
+      - 'subprojects/**'
+      - 'meson.build'
+      - 'meson_options.txt'
+      - '.github/workflows/benchmark.yml'
+      - '*.cfg'
   workflow_dispatch:
 
 env:
@@ -36,15 +47,15 @@ jobs:
 
       - name: Lint cpplint-cpp
         run: |
-          python benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "python cpplint.py" .
-          sh memory_usage.sh "./build/cpplint-cpp" .
+          python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
+          sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" .
+          sh ./benchmark/memory_usage.sh "python cpplint.py" .
 
       - name: Lint googletest
         run: |
-          python benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
-          sh memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }}
+          python ./benchmark/benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp"
+          sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }}
+          sh ./benchmark/memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
 
       - name: Check required GLIBC version
         run: bash .github/workflows/check_glibc_compatibility.sh ./build/cpplint-cpp

benchmark.py → benchmark/benchmark.py

@@ -1,9 +1,11 @@
+# Script to measure the average execution time for cpplint-cpp and cpplint.py.
+
 import argparse
 import os
 import subprocess
 import time
 
-def measure_time(command, repeat_time=30):
+def measure_time(command, repeat_time=60):
     duration = 0
     count = 0
     while(duration < repeat_time):
@@ -30,8 +32,8 @@ def get_args():
                         help="command to run cpplint.py")
     parser.add_argument("--options", default="--recursive --quiet --counting=detailed", type=str,
                         help="options for cpplint")
-    parser.add_argument("--time", default=30, type=int,
-                        help="Minimum measurement time for a command. Default to 30 (sec)")
+    parser.add_argument("--time", default=60, type=int,
+                        help="Minimum measurement time for a command. Default to 60 (sec)")
     return parser.parse_args()
 
 

memory_usage.sh → benchmark/memory_usage.sh

@@ -2,10 +2,10 @@
 # Script to measure memory usage with cpplint
 #
 # Examples
-#  sh memory_usage.sh "python3 cpplint.py" .
-#  sh memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0
+#  sh ./benchmark/memory_usage.sh "python3 cpplint.py" .
+#  sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0
 
-echo Measuring memory usage...
+echo Measuring memory usage: $1
 kib="$(/usr/bin/time -f "%M" sh -c "$1 --recursive --quiet --counting=detailed $2 >/dev/null 2>&1 || exit 0" 2>&1)"
 mib=$(echo "scale=2; $kib / 1024" | bc)
 echo Maximum memory usage: ${mib} MiB

docs/BENCHMARK.md

@@ -0,0 +1,36 @@
+# Benchmarking
+
+This repository contains some scripts for benchmarking.
+
+## Execution time
+
+[`benchmark.py`](../benchmark/benchmark.py) can measure the average execution time for cpplint-cpp and cpplint.py. It takes about two minutes for measurement.
+
+```console
+$ python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
+Measuring time for cpplint-cpp: ./build/cpplint-cpp --recursive --quiet --counting=detailed .
+Measuring time for cpplint.py: python cpplint.py --recursive --quiet --counting=detailed .
+Execution time for cpplint-cpp: x.xxxxxx seconds
+Execution time for cpplint.py: x.xxxxxx seconds
+```
+
+## Memory usage
+
+[`memory_usage.sh`](../benchmark/memory_usage.sh) can measure memory usage for a linter against a directory.
+
+```console
+$ sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" .
+Measuring memory usage: ./build/cpplint-cpp
+Maximum memory usage: xx.xx MiB
+
+$ sh ./benchmark/memory_usage.sh "python cpplint.py" .
+Measuring memory usage: python cpplint.py
+Maximum memory usage: xx.xx MiB
+```
+
+## Github Actions
+
+You don't need to setup environment for benchmarking.
+You can use [`benchmark.yml`](../.github/workflows/benchmark.yml) in your fork to run `benchmark.py` and `memory_usage.sh`.  
+
+![benchmark.yml](https://github.com/user-attachments/assets/491f4c4b-32c4-4362-9406-2b1816e51574)

docs/README.md

@@ -21,7 +21,7 @@ pip install cpplint-cpp --no-index --find-links https://matyalatte.github.io/cpp
 
 Here is an analysis of the performance differences between `cpplint-cpp` and `cpplint.py` against two repositories:
 [`googletest`](https://github.com/google/googletest) and `cpplint-cpp`.
-Measurements were taken on an Ubuntu runner with [`benchmark.yml`](../.github/workflows/benchmark.yml).
+Measurements were taken on an Ubuntu runner with [some scripts](BENCHMARK.md).
 
 ### Execution time