Skip to content

Commit 14373f6

Browse files
[Test] Remove V0 accuracy test and enable MoE and VL test on V1 (#1574)
### What this PR does / why we need it? Update accuracy test 1. remove accuarcy report on V0 2. add parallel and execution mode 3. add Qwen/Qwen3-30B-A3B and remove Qwen/Qwen2.5-7B-Instruct ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passed Signed-off-by: hfadzxy <starmoon_zhang@163.com>
1 parent 0c1d239 commit 14373f6

File tree

2 files changed

+152
-114
lines changed

2 files changed

+152
-114
lines changed

.github/workflows/accuracy_test.yaml

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@ on:
5353
type: choice
5454
options:
5555
- all
56-
- Qwen/Qwen2.5-7B-Instruct
5756
- Qwen/Qwen2.5-VL-7B-Instruct
5857
- Qwen/Qwen3-8B-Base
58+
- Qwen/Qwen3-30B-A3B
5959
default: 'all'
6060

6161
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
@@ -77,48 +77,48 @@ jobs:
7777
${{
7878
(contains(github.event.pull_request.labels.*.name, 'accuracy-test') ||
7979
contains(github.event.pull_request.labels.*.name, 'vl-accuracy-test') ||
80+
contains(github.event.pull_request.labels.*.name, 'moe-accuracy-test') ||
8081
contains(github.event.pull_request.labels.*.name, 'dense-accuracy-test')) &&
8182
contains(github.event.pull_request.labels.*.name, 'ready-for-test') ||
8283
github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
8384
}}
8485
runs-on: >-
8586
${{
86-
(matrix.model_name == 'Qwen/Qwen2.5-VL-7B-Instruct' && 'linux-arm64-npu-4') ||
87+
(matrix.model_name == 'Qwen/Qwen3-30B-A3B' && 'linux-arm64-npu-4') ||
8788
'linux-arm64-npu-2'
8889
}}
8990
strategy:
9091
matrix:
91-
vllm_use_version: [0, 1]
92+
vllm_use_version: [1]
9293
# the accuracy test will run:
9394
# 1. workflow_dispatch with models input
94-
# - all: Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct, Qwen/Qwen3-8B-Base
95-
# - specified but not all: Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct, Qwen/Qwen3-8B-Base
95+
# - all: Qwen/Qwen3-30B-A3B, Qwen/Qwen2.5-VL-7B-Instruct, Qwen/Qwen3-8B-Base
96+
# - specified but not all: Qwen/Qwen3-30B-A3B, Qwen/Qwen2.5-VL-7B-Instruct, Qwen/Qwen3-8B-Base
9697
# 2. PR labeled with "*-accuracy-test"
97-
# - accuracy-test: Qwen/Qwen2.5-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct
98-
# - dense-accuracy-test: Qwen/Qwen2.5-7B-Instruct
98+
# - accuracy-test: Qwen/Qwen3-8B-Base, Qwen/Qwen2.5-VL-7B-Instruct, Qwen/Qwen3-30B-A3B
99+
# - dense-accuracy-test: Qwen/Qwen3-8B-Base
99100
# - vl-accuracy-test: Qwen/Qwen2.5-VL-7B-Instruct
101+
# - moe-accuracy-test: Qwen/Qwen3-30B-A3B
100102
model_name: ${{ fromJSON(
101103
(github.event_name == 'schedule' &&
102-
'["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
104+
'["Qwen/Qwen3-30B-A3B","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
103105
(github.event.inputs.models == 'all' &&
104-
'["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
105-
(github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' &&
106-
'["Qwen/Qwen2.5-7B-Instruct"]') ||
106+
'["Qwen/Qwen3-30B-A3B","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
107+
(github.event.inputs.models == 'Qwen/Qwen3-30B-A3B' &&
108+
'["Qwen/Qwen3-30B-A3B"]') ||
107109
(github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' &&
108110
'["Qwen/Qwen2.5-VL-7B-Instruct"]') ||
109111
(github.event.inputs.models == 'Qwen/Qwen3-8B-Base' &&
110112
'["Qwen/Qwen3-8B-Base"]') ||
111113
contains(github.event.pull_request.labels.*.name, 'accuracy-test') &&
112-
'["Qwen/Qwen3-8B-Base","Qwen/Qwen2.5-VL-7B-Instruct"]' ||
114+
'["Qwen/Qwen3-8B-Base","Qwen/Qwen2.5-VL-7B-Instruct", "Qwen/Qwen3-30B-A3B"]' ||
113115
contains(github.event.pull_request.labels.*.name, 'dense-accuracy-test') &&
114116
'["Qwen/Qwen3-8B-Base"]' ||
115117
contains(github.event.pull_request.labels.*.name, 'vl-accuracy-test') &&
116-
'["Qwen/Qwen2.5-VL-7B-Instruct"]'
118+
'["Qwen/Qwen2.5-VL-7B-Instruct"]' ||
119+
contains(github.event.pull_request.labels.*.name, 'moe-accuracy-test') &&
120+
'["Qwen/Qwen3-30B-A3B"]'
117121
) }}
118-
# Remove exclude after https://github.com/vllm-project/vllm-ascend/issues/1044 resolved
119-
exclude:
120-
- model_name: Qwen/Qwen2.5-VL-7B-Instruct
121-
vllm_use_version: 1
122122

123123
fail-fast: false
124124
name: ${{ matrix.model_name }} accuracy V${{ matrix.vllm_use_version }}
@@ -187,23 +187,19 @@ jobs:
187187
- name: Get vLLM commit hash and URL
188188
working-directory: ./vllm-empty
189189
run: |
190-
VLLM_COMMIT=$(git rev-parse HEAD)
190+
VLLM_COMMIT=$(git rev-parse --short=7 HEAD)
191191
echo "VLLM_COMMIT=$VLLM_COMMIT" >> $GITHUB_ENV
192-
echo "VLLM_COMMIT_URL=https://github.com/vllm-project/vllm/commit/$VLLM_COMMIT" >> $GITHUB_ENV
193192
194193
- name: Get vLLM-Ascend commit hash and URL
195194
working-directory: ./vllm-ascend
196195
run: |
197-
VLLM_ASCEND_COMMIT=$(git rev-parse HEAD)
196+
VLLM_ASCEND_COMMIT=$(git rev-parse --short=7 HEAD)
198197
echo "VLLM_ASCEND_COMMIT=$VLLM_ASCEND_COMMIT" >> $GITHUB_ENV
199-
echo "VLLM_ASCEND_COMMIT_URL=https://github.com/vllm-project/vllm-ascend/commit/$VLLM_ASCEND_COMMIT" >> $GITHUB_ENV
200198
201-
- name: Print resolved hashes and URLs
199+
- name: Print resolved hashes
202200
run: |
203201
echo "vLLM : ${{ env.VLLM_COMMIT }}"
204-
echo "vLLM link : ${{ env.VLLM_COMMIT_URL }}"
205202
echo "vLLM-Ascend: ${{ env.VLLM_ASCEND_COMMIT }}"
206-
echo "Ascend link: ${{ env.VLLM_ASCEND_COMMIT_URL }}"
207203
208204
- name: Install lm-eval, ray, and datasets
209205
run: |
@@ -262,8 +258,6 @@ jobs:
262258
--vllm_version "${{ env.GHA_VLLM_VERSION }}" \
263259
--vllm_commit "${{ env.VLLM_COMMIT }}" \
264260
--vllm_ascend_commit "${{ env.VLLM_ASCEND_COMMIT }}" \
265-
--vllm_commit_url "${{ env.VLLM_COMMIT_URL }}" \
266-
--vllm_ascend_commit_url "${{ env.VLLM_ASCEND_COMMIT_URL }}" \
267261
--vllm_use_v1 "$VLLM_USE_V1"
268262
269263
- name: Generate step summary
@@ -385,7 +379,7 @@ jobs:
385379
body: `The accuracy results running on NPU Altlas A2 have changed, updating reports for:
386380
${{
387381
github.event.inputs.models == 'all'
388-
&& 'All models (Qwen2.5-7B-Instruct, Qwen2.5-VL-7B-Instruct, Qwen3-8B-Base)'
382+
&& 'All models (Qwen/Qwen3-30B-A3B, Qwen2.5-VL-7B-Instruct, Qwen3-8B-Base)'
389383
|| github.event.inputs.models
390384
}}
391385

0 commit comments

Comments
 (0)