Skip to content

Commit 4a2505f

Browse files
[accuracy test]Update cann version and huggingface-hub version for Qwen3 (#823)
### What this PR does / why we need it? 1. update cann version to 8.1.0 for multimodal 2. fix huggingface-hub version to adapt to qwen3 3. change Qwen3-8B to Qwen-8B-Base, Signed-off-by: hfadzxy <starmoon_zhang@163.com>
1 parent 5305a2c commit 4a2505f

File tree

2 files changed

+15
-21
lines changed

2 files changed

+15
-21
lines changed

.github/workflows/accuracy_report.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,12 @@ jobs:
7070
env:
7171
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
7272

73-
- name: Query artifact run id for Qwen3-8B V0 latest artifact
74-
id: get_Qwen3_8B_latest_run_id_V0
73+
- name: Query artifact run id for Qwen3-8B-Base V0 latest artifact
74+
id: get_Qwen3_8B_Base_latest_run_id_V0
7575
run: |
7676
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
7777
RUN_ID=$(echo "$ARTIFACT_JSON" | \
78-
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
78+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
7979
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
8080
env:
8181
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -107,22 +107,22 @@ jobs:
107107
repository: vllm-project/vllm-ascend
108108
run-id: ${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
109109

110-
- name: Download Qwen/Qwen3-8B Artifact
110+
- name: Download Qwen/Qwen3-8B-Base Artifact
111111
uses: actions/download-artifact@v4
112112
with:
113-
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-V0-report
113+
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report
114114
path: ./docs/source/developer_guide/evaluation/accuracy_report
115115
github-token: ${{ secrets.GITHUB_TOKEN }}
116116
repository: vllm-project/vllm-ascend
117-
run-id: ${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }}
117+
run-id: ${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}
118118

119119
- name: Display Files
120120
working-directory: ./docs/source/developer_guide/evaluation/accuracy_report
121121
run: |
122122
cat ./Qwen2.5-VL-7B-Instruct.md
123123
cat ./Llama-3.1-8B-Instruct.md
124124
cat ./Qwen2.5-7B-Instruct.md
125-
cat ./Qwen3-8B.md
125+
cat ./Qwen3-8B-Base.md
126126
127127
- name: Create Pull Request for markdown update
128128
uses: peter-evans/create-pull-request@v7
@@ -141,10 +141,10 @@ jobs:
141141
- [Qwen2.5-7B-Instruct accuracy report][2]
142142
- [Llama-3.1-8B-Instruct accuracy report][3]
143143
- [Qwen2.5-VL-7B-Instruct accuracy report][4]
144-
- [Qwen3-8B accuracy report][5]
144+
- [Qwen3-8B-Base accuracy report][5]
145145
146146
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
147147
[2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
148148
[3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
149149
[4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}
150-
[5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_latest_run_id_V0.outputs.runid }}
150+
[5]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}

.github/workflows/accuracy_test.yaml

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,15 @@ on:
2929
required: true
3030
type: string
3131
models:
32-
description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B)'
32+
description: 'choose model(all/Qwen2.5-7B-Instruct/Llama-3.1-8B-Instruct/Qwen2.5-VL-7B-Instruct/Qwen3-8B-Base)'
3333
required: true
3434
type: choice
3535
options:
3636
- all
3737
- Qwen/Qwen2.5-7B-Instruct
3838
- meta-llama/Llama-3.1-8B-Instruct
3939
- Qwen/Qwen2.5-VL-7B-Instruct
40-
- Qwen/Qwen3-8B
40+
- Qwen/Qwen3-8B-Base
4141
default: 'all'
4242

4343
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
@@ -54,16 +54,16 @@ jobs:
5454
strategy:
5555
matrix:
5656
include: ${{ fromJSON(
57-
(github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]') ||
57+
(github.event.inputs.models == 'all' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"},{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"},{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}, {"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]') ||
5858
(github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-7B-Instruct","output_file":"Qwen2.5-7B-Instruct"}]') ||
5959
(github.event.inputs.models == 'meta-llama/Llama-3.1-8B-Instruct' && '[{"model_name":"meta-llama/Llama-3.1-8B-Instruct","output_file":"Llama-3.1-8B-Instruct"}]') ||
6060
(github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' && '[{"model_name":"Qwen/Qwen2.5-VL-7B-Instruct","output_file":"Qwen2.5-VL-7B-Instruct"}]') ||
61-
(github.event.inputs.models == 'Qwen/Qwen3-8B' && '[{"model_name":"Qwen/Qwen3-8B","output_file":"Qwen3-8B"}]')
61+
(github.event.inputs.models == 'Qwen/Qwen3-8B-Base' && '[{"model_name":"Qwen/Qwen3-8B-Base","output_file":"Qwen3-8B-Base"}]')
6262
) }}
6363
fail-fast: false
6464

6565
container:
66-
image: quay.io/ascend/cann:8.0.0-910b-ubuntu22.04-py3.10
66+
image: m.daocloud.io/quay.io/ascend/cann:8.1.rc1-910b-ubuntu22.04-py3.10
6767
env:
6868
HF_ENDPOINT: https://hf-mirror.com
6969
HF_TOKEN: ${{ secrets.HF_TOKEN }}
@@ -86,12 +86,6 @@ jobs:
8686
apt install git -y
8787
git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
8888
89-
- name: Install system dependencies
90-
run: |
91-
apt-get -y install `cat packages.txt`
92-
apt-get -y install gcc g++ cmake libnuma-dev
93-
94-
9589
- name: Install system dependencies
9690
run: |
9791
apt-get -y install `cat packages.txt`
@@ -147,7 +141,7 @@ jobs:
147141
working-directory: ./lm-eval
148142
run: |
149143
pip install -e .
150-
pip install ray datasets==2.16.0 transformers==4.50.3 huggingface-hub==0.29.3
144+
pip install ray datasets==2.16.0
151145
152146
- name: Collect version info
153147
run: |

0 commit comments

Comments
 (0)