Skip to content

Commit 31dd471

Browse files
authored
[CI] Add workflow_dispatch and use main benchmarks directly (#1071)
### What this PR does / why we need it? This is for the benchmark iteration, which will change the benchmark scripts while checkouting each commit. So we need ensure the benchmark scripts always available. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Manaully --------- Signed-off-by: wangli <wangli858794774@gmail.com>
1 parent 9e855b7 commit 31dd471

File tree

1 file changed

+22
-9
lines changed

1 file changed

+22
-9
lines changed

.github/workflows/nightly_benchmarks.yaml

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ on:
2222
schedule:
2323
# Run at 02:00 everyday
2424
- cron: '00 18 * * *'
25+
26+
workflow_dispatch:
27+
# Allow manual triggering of the workflow
2528

2629
pull_request:
2730
types: [ labeled ]
@@ -40,7 +43,7 @@ concurrency:
4043

4144
jobs:
4245
test:
43-
if: ${{ contains(github.event.pull_request.labels.*.name, 'performance-test') && contains(github.event.pull_request.labels.*.name, 'ready-for-test') || github.event_name == 'schedule' }}
46+
if: ${{ contains(github.event.pull_request.labels.*.name, 'performance-test') && contains(github.event.pull_request.labels.*.name, 'ready-for-test') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
4447

4548
name: Benchmarks/vLLM=${{ matrix.vllm_branch }}, vLLM-Ascend=${{ matrix.vllm_ascend_branch }}
4649
runs-on: 'linux-arm64-npu-static-8'
@@ -111,7 +114,7 @@ jobs:
111114
pip install -r benchmarks/requirements-bench.txt
112115
113116
- name: Run current commit benchmarks
114-
if: github.event_name != 'schedule'
117+
if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
115118
run: |
116119
# Sometimes we only want to run benchmarks on the current commit
117120
# This is useful for debugging or a release benchmark
@@ -120,12 +123,12 @@ jobs:
120123
python3 benchmarks/scripts/convert_json_to_markdown.py
121124
122125
- name: Generate step summary
123-
if: github.event_name != 'schedule'
126+
if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
124127
run: |
125128
cat ./benchmarks/results/benchmark_results.md >> $GITHUB_STEP_SUMMARY
126129
127130
- name: Upload benchmark artifacts
128-
if: github.event_name != 'schedule'
131+
if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
129132
uses: actions/upload-artifact@v4
130133
with:
131134
name: "benchmark-performance-${{ matrix.vllm_branch }}-${{ matrix.vllm_ascend_branch }}-report"
@@ -135,19 +138,28 @@ jobs:
135138
overwrite: true
136139

137140
- name: Install elastic_tool
138-
if: github.event_name == 'schedule'
141+
if: github.event_name != 'pull_request'
139142
run: |
140143
pip install escli-tool==0.2.1
141144
142145
- name: Collect pr info from vllm-project/vllm-ascend
143-
if: github.event_name == 'schedule'
146+
if: github.event_name != 'pull_request'
144147
run: |
145148
# Only get the pull request which may influences performance
146-
git log --pretty=format:"%H %s" -- '**/*.py' ':!docs/*' ':!tests/*' ':!examples/*' > commit_log.txt
149+
git log --pretty=format:"%H %s" -- '**/*.py' ':!docs/*' ':!tests/*' ':!examples/*' ':!benchmarks/*' > commit_log.txt
147150
escli check commit_log.txt
151+
152+
- name: Prepare benchmark script in advance
153+
if: github.event_name != 'pull_request'
154+
# This is for the benchmark iteration, which will change the benchmark scripts while checkouting each commit.
155+
# We need ensure the benchmark scripts always available.
156+
run: |
157+
# Prepare the benchmark script in advance
158+
mkdir -p /github/home/benchmarks
159+
cp -r benchmarks/* /github/home/benchmarks/
148160
149161
- name: Run benchmark iteration
150-
if: github.event_name == 'schedule'
162+
if: github.event_name != 'pull_request'
151163
run: |
152164
while IFS= read -r line || [[ -n "$line" ]]; do
153165
commit_id=${line%% *}
@@ -165,8 +177,9 @@ jobs:
165177
echo "vllm branch: ${{ matrix.vllm_branch }}"
166178
echo "vllm-ascend branch: ${{ matrix.vllm_ascend_branch }}"
167179
echo "------------------------"
168-
180+
cd /github/home
169181
bash benchmarks/scripts/run-performance-benchmarks.sh
182+
cd -
170183
# send the result to es
171184
if [[ "${{ github.event_name }}" != "pull request" ]]; then
172185
escli add --vllm_branch ${{ matrix.vllm_branch }} \

0 commit comments

Comments
 (0)