From e57d1d151ce463800ef35801245a6c42c27b32d2 Mon Sep 17 00:00:00 2001
From: matyalatte <matyalatte@gmail.com>
Date: Sat, 7 Sep 2024 15:52:30 +0900
Subject: [PATCH 1/3] improve benchmark.yml

- Run benchmark.yml when pushing PRs.
- Extend measuring time from 30 sec to 1 min.
- Show executable path in memory_usage.sh.
- Run cpplint-cpp first when measuring memory usage.
---
 .github/workflows/benchmark.yml | 16 ++++++++++++++--
 benchmark.py                    |  6 +++---
 memory_usage.sh                 |  2 +-
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 0c990c9..987e86f 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,6 +1,18 @@
 name: Benchmark
 
 on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'include/**'
+      - 'src/**'
+      - 'tests/**'
+      - 'subprojects/**'
+      - 'meson.build'
+      - 'meson_options.txt'
+      - '.github/workflows/ci.yml'
+      - '*.cfg'
   workflow_dispatch:
 
 env:
@@ -37,14 +49,14 @@ jobs:
       - name: Lint cpplint-cpp
         run: |
           python benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "python cpplint.py" .
           sh memory_usage.sh "./build/cpplint-cpp" .
+          sh memory_usage.sh "python cpplint.py" .
 
       - name: Lint googletest
         run: |
           python benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
           sh memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }}
+          sh memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
 
       - name: Check required GLIBC version
         run: bash .github/workflows/check_glibc_compatibility.sh ./build/cpplint-cpp
diff --git a/benchmark.py b/benchmark.py
index ed3b22c..be79aea 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -3,7 +3,7 @@
 import subprocess
 import time
 
-def measure_time(command, repeat_time=30):
+def measure_time(command, repeat_time=60):
     duration = 0
     count = 0
     while(duration < repeat_time):
@@ -30,8 +30,8 @@ def get_args():
                         help="command to run cpplint.py")
     parser.add_argument("--options", default="--recursive --quiet --counting=detailed", type=str,
                         help="options for cpplint")
-    parser.add_argument("--time", default=30, type=int,
-                        help="Minimum measurement time for a command. Default to 30 (sec)")
+    parser.add_argument("--time", default=60, type=int,
+                        help="Minimum measurement time for a command. Default to 60 (sec)")
     return parser.parse_args()
 
 
diff --git a/memory_usage.sh b/memory_usage.sh
index 2f08fbe..5fdc0af 100644
--- a/memory_usage.sh
+++ b/memory_usage.sh
@@ -5,7 +5,7 @@
 #  sh memory_usage.sh "python3 cpplint.py" .
 #  sh memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0
 
-echo Measuring memory usage...
+echo Measuring memory usage: $1
 kib="$(/usr/bin/time -f "%M" sh -c "$1 --recursive --quiet --counting=detailed $2 >/dev/null 2>&1 || exit 0" 2>&1)"
 mib=$(echo "scale=2; $kib / 1024" | bc)
 echo Maximum memory usage: ${mib} MiB

From cd7246cc087b7b0682c67db8cfd8c294cd7a3604 Mon Sep 17 00:00:00 2001
From: matyalatte <matyalatte@gmail.com>
Date: Sat, 7 Sep 2024 16:59:08 +0900
Subject: [PATCH 2/3] add BENCHMARK.md and a directory for benchmark scripts

---
 .github/workflows/benchmark.yml              | 12 +++----
 benchmark.py => benchmark/benchmark.py       |  2 ++
 memory_usage.sh => benchmark/memory_usage.sh |  4 +--
 docs/BENCHMARK.md                            | 36 ++++++++++++++++++++
 docs/README.md                               |  2 +-
 5 files changed, 47 insertions(+), 9 deletions(-)
 rename benchmark.py => benchmark/benchmark.py (96%)
 rename memory_usage.sh => benchmark/memory_usage.sh (68%)
 create mode 100644 docs/BENCHMARK.md

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 987e86f..4e5b899 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -48,15 +48,15 @@ jobs:
 
       - name: Lint cpplint-cpp
         run: |
-          python benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "./build/cpplint-cpp" .
-          sh memory_usage.sh "python cpplint.py" .
+          python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
+          sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" .
+          sh ./benchmark/memory_usage.sh "python cpplint.py" .
 
       - name: Lint googletest
         run: |
-          python benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp"
-          sh memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }}
-          sh memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
+          python ./benchmark/benchmark.py ../googletest-${{ env.GTEST_VER }} --cpplint_cpp="./build/cpplint-cpp"
+          sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-${{ env.GTEST_VER }}
+          sh ./benchmark/memory_usage.sh "python cpplint.py" ../googletest-${{ env.GTEST_VER }}
 
       - name: Check required GLIBC version
         run: bash .github/workflows/check_glibc_compatibility.sh ./build/cpplint-cpp
diff --git a/benchmark.py b/benchmark/benchmark.py
similarity index 96%
rename from benchmark.py
rename to benchmark/benchmark.py
index be79aea..0f48886 100644
--- a/benchmark.py
+++ b/benchmark/benchmark.py
@@ -1,3 +1,5 @@
+# Script to measure the average execution time for cpplint-cpp and cpplint.py.
+
 import argparse
 import os
 import subprocess
diff --git a/memory_usage.sh b/benchmark/memory_usage.sh
similarity index 68%
rename from memory_usage.sh
rename to benchmark/memory_usage.sh
index 5fdc0af..0e3f3d8 100644
--- a/memory_usage.sh
+++ b/benchmark/memory_usage.sh
@@ -2,8 +2,8 @@
 # Script to measure memory usage with cpplint
 #
 # Examples
-#  sh memory_usage.sh "python3 cpplint.py" .
-#  sh memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0
+#  sh ./benchmark/memory_usage.sh "python3 cpplint.py" .
+#  sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" ../googletest-1.4.0
 
 echo Measuring memory usage: $1
 kib="$(/usr/bin/time -f "%M" sh -c "$1 --recursive --quiet --counting=detailed $2 >/dev/null 2>&1 || exit 0" 2>&1)"
diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md
new file mode 100644
index 0000000..0821ce7
--- /dev/null
+++ b/docs/BENCHMARK.md
@@ -0,0 +1,36 @@
+# Benchmarking
+
+This repository contains some scripts for benchmarking.
+
+## Execution time
+
+[`benchmark.py`](../benchmark/benchmark.py) can measure the average execution time for cpplint-cpp and cpplint.py. It takes about two minutes for measurement.
+
+```console
+$ python ./benchmark/benchmark.py . --cpplint_cpp="./build/cpplint-cpp"
+Measuring time for cpplint-cpp: ./build/cpplint-cpp --recursive --quiet --counting=detailed .
+Measuring time for cpplint.py: python cpplint.py --recursive --quiet --counting=detailed .
+Execution time for cpplint-cpp: x.xxxxxx seconds
+Execution time for cpplint.py: x.xxxxxx seconds
+```
+
+## Memory usage
+
+[`memory_usage.sh`](../benchmark/memory_usage.sh) can measure memory usage for a linter against a directory.
+
+```console
+$ sh ./benchmark/memory_usage.sh "./build/cpplint-cpp" .
+Measuring memory usage: ./build/cpplint-cpp
+Maximum memory usage: xx.xx MiB
+
+$ sh ./benchmark/memory_usage.sh "python cpplint.py" .
+Measuring memory usage: python cpplint.py
+Maximum memory usage: xx.xx MiB
+```
+
+## Github Actions
+
+You don't need to setup environment for benchmarking.
+You can use [`benchmark.yml`](../.github/workflows/benchmark.yml) in your fork to run `benchmark.py` and `memory_usage.sh`.  
+
+![benchmark.yml](https://github.com/user-attachments/assets/491f4c4b-32c4-4362-9406-2b1816e51574)
diff --git a/docs/README.md b/docs/README.md
index 5d6d271..c44fee2 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -21,7 +21,7 @@ pip install cpplint-cpp --no-index --find-links https://matyalatte.github.io/cpp
 
 Here is an analysis of the performance differences between `cpplint-cpp` and `cpplint.py` against two repositories:
 [`googletest`](https://github.com/google/googletest) and `cpplint-cpp`.
-Measurements were taken on an Ubuntu runner with [`benchmark.yml`](../.github/workflows/benchmark.yml).
+Measurements were taken on an Ubuntu runner with [some scripts](BENCHMARK.md).
 
 ### Execution time
 

From 110db1439376e41f47d1cff4afcc64f485d064b5 Mon Sep 17 00:00:00 2001
From: matyalatte <matyalatte@gmail.com>
Date: Fri, 13 Sep 2024 22:23:22 +0900
Subject: [PATCH 3/3] fix trigger for benchmark.yml

---
 .github/workflows/benchmark.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 4e5b899..955750a 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -7,11 +7,10 @@ on:
     paths:
       - 'include/**'
       - 'src/**'
-      - 'tests/**'
       - 'subprojects/**'
       - 'meson.build'
       - 'meson_options.txt'
-      - '.github/workflows/ci.yml'
+      - '.github/workflows/benchmark.yml'
       - '*.cfg'
   workflow_dispatch: