Skip to content

Commit 030fe89

Browse files
momo60922dimensionsYikunzhangxinyuehfadwangxiyuan
authored
[cherry-pick][0.9.1] rebase main (#1250)
<!-- Thanks for sending a pull request! --> ### What this PR does / why we need it? rebase main ### Does this PR introduce _any_ user-facing change? <!-- Note that it means *any* user-facing change including all aspects such as API, interface or other behavior changes. Documentation-only updates are not considered user-facing changes. --> ### How was this patch tested? <!-- CI passed with new added/existing test. If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future. If tests were not added, please describe why they were not added and/or why it was difficult to add. --> --------- Signed-off-by: 22dimensions <waitingwind@foxmail.com> Signed-off-by: wangxiaoxin (A) <wangxiaoxin7@huawei.com> Signed-off-by: Yikun Jiang <yikunkero@gmail.com> Signed-off-by: hfadzxy <starmoon_zhang@163.com> Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: MengqingCao <cmq0113@163.com> Signed-off-by: sdmyzlp <lrwei2@petalmail.com> Signed-off-by: depeng1994 <depengzhang@foxmail.com> Signed-off-by: ttanzhiqiang <389825161@qq.com> Signed-off-by: yzim <43207690+yzim@users.noreply.github.com> Signed-off-by: chenwaner <861645847@qq.com> Signed-off-by: whx-sjtu <2952154980@qq.com> Signed-off-by: wangli <wangli858794774@gmail.com> Signed-off-by: wangyanhui-cmss <wangyanhui_yewu@cmss.chinamobile.com> Signed-off-by: wan_danfeng <wonderful199082@126.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Signed-off-by: zhuo97 <1103045176@qq.com> Co-authored-by: 22dimensions <waitingwind@foxmail.com> Co-authored-by: Yikun Jiang <yikunkero@gmail.com> Co-authored-by: zhangxinyuehfad <59153331+zhangxinyuehfad@users.noreply.github.com> Co-authored-by: wangxiyuan <wangxiyuan1007@gmail.com> Co-authored-by: Mengqing Cao <cmq0113@163.com> Co-authored-by: sdmyzlp <117554856+sdmyzlp@users.noreply.github.com> Co-authored-by: depeng1994 <166494784+depeng1994@users.noreply.github.com> Co-authored-by: ttanzhiqiang <38750855+ttanzhiqiang@users.noreply.github.com> Co-authored-by: yz <43207690+yzim@users.noreply.github.com> Co-authored-by: chenwaner <48718746+chenwaner@users.noreply.github.com> Co-authored-by: whx <56632993+whx-sjtu@users.noreply.github.com> Co-authored-by: Li Wang <wangli858794774@gmail.com> Co-authored-by: wangyanhui-cmss <wangyanhui_yewu@cmss.chinamobile.com> Co-authored-by: Wan_Danfeng <wonderful199082@126.com> Co-authored-by: zhuo97 <49392868+zhuo97@users.noreply.github.com> Co-authored-by: wangxiaoxin (A) <wangxiaoxin7@huawei.com>
1 parent f6d033b commit 030fe89

File tree

67 files changed

+2871
-1135
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+2871
-1135
lines changed

‎.github/workflows/accuracy_report.yaml

Lines changed: 149 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -19,110 +19,184 @@ name: Accuracy Report
1919
on:
2020
workflow_dispatch:
2121
inputs:
22-
branch:
23-
description: 'choose a dev branch to pr'
22+
vllm-ascend-branch:
23+
description: 'vllm-ascend branch:'
2424
required: true
25-
vllm-ascend-version:
26-
description: 'what vllm-ascend version to accuracy test?'
25+
type: choice
26+
options:
27+
- main
28+
- v0.7.3-dev
29+
models:
30+
description: 'models:'
2731
required: true
28-
type: string
32+
type: choice
33+
options:
34+
- all
35+
- Qwen/Qwen2.5-7B-Instruct
36+
- Qwen/Qwen2.5-VL-7B-Instruct
37+
- Qwen/Qwen3-8B-Base
38+
default: 'all'
39+
2940
jobs:
30-
download:
41+
download_reports:
3142
runs-on: ubuntu-latest
43+
strategy:
44+
matrix:
45+
model: ${{ fromJSON(
46+
(github.event.inputs.models == 'all' &&
47+
'["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
48+
(github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' &&
49+
'["Qwen/Qwen2.5-7B-Instruct"]') ||
50+
(github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' &&
51+
'["Qwen/Qwen2.5-VL-7B-Instruct"]') ||
52+
(github.event.inputs.models == 'Qwen/Qwen3-8B-Base' &&
53+
'["Qwen/Qwen3-8B-Base"]')
54+
) }}
55+
56+
version: [0, 1]
57+
exclude:
58+
- model: 'Qwen/Qwen2.5-VL-7B-Instruct'
59+
version: 1
60+
fail-fast: false
61+
62+
name: Download ${{ matrix.model }} V${{ matrix.version }}
3263
steps:
3364
- name: Checkout repository
3465
uses: actions/checkout@v4
3566
with:
36-
ref: ${{ github.event.inputs.branch }}
37-
38-
- name: Debug List Artifacts
39-
run: gh api /repos/${{ github.repository }}/actions/artifacts
40-
env:
41-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
67+
ref: ${{ github.event.inputs.vllm-ascend-branch }}
4268

43-
- name: Query artifact run id for Qwen2.5-VL-7B-Instruct V0 latest artifact
44-
id: get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0
69+
- name: Get base model name
70+
id: get_basename
4571
run: |
46-
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
47-
RUN_ID=$(echo "$ARTIFACT_JSON" | \
48-
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-VL-7B-Instruct-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
49-
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
50-
env:
51-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72+
model_base_name=$(basename "${{ matrix.model }}")
73+
echo "model_base_name=$model_base_name" >> $GITHUB_OUTPUT
74+
shell: bash
5275

53-
- name: Query artifact run id for Qwen2.5-7B-Instruct V0 latest artifact
54-
id: get_Qwen2_5_7B_Instruct_latest_run_id_V0
76+
- name: Query artifact run id
77+
id: get_run_id
5578
run: |
56-
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
79+
ARTIFACT_PATTERN="${{ github.event.inputs.vllm-ascend-branch }}-${{ steps.get_basename.outputs.model_base_name }}-V${{ matrix.version }}-report"
80+
echo "Querying artifacts with pattern: $ARTIFACT_PATTERN"
81+
82+
ARTIFACT_JSON=$(gh api --paginate /repos/${{ github.repository }}/actions/artifacts || echo "{}")
83+
5784
RUN_ID=$(echo "$ARTIFACT_JSON" | \
58-
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-7B-Instruct-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
59-
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
85+
jq -s -r --arg pattern "$ARTIFACT_PATTERN" \
86+
'[.[].artifacts[]] | map(select(.name | test($pattern))) | sort_by(.created_at) | last | .workflow_run.id // empty')
87+
88+
if [ -z "$RUN_ID" ]; then
89+
echo "::warning::No artifact found matching pattern $ARTIFACT_PATTERN. Skipping download."
90+
echo "runid=" >> $GITHUB_OUTPUT
91+
else
92+
echo "Found matching artifact with run ID: $RUN_ID"
93+
echo "runid=$RUN_ID" >> $GITHUB_OUTPUT
94+
fi
6095
env:
61-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
96+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
6297

63-
- name: Query artifact run id for Qwen3-8B-Base V0 latest artifact
64-
id: get_Qwen3_8B_Base_latest_run_id_V0
65-
run: |
66-
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
67-
RUN_ID=$(echo "$ARTIFACT_JSON" | \
68-
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
69-
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
70-
env:
71-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72-
73-
- name: Download Qwen/Qwen2.5-VL-7B-Instruct V0 Artifact
98+
- name: Download Artifact
99+
if: ${{ steps.get_run_id.outputs.runid != '' }}
74100
uses: actions/download-artifact@v4
75101
with:
76-
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-VL-7B-Instruct-V0-report
77-
path: ./docs/source/developer_guide/evaluation/accuracy_report
78-
github-token: ${{ secrets.GITHUB_TOKEN }}
79-
repository: vllm-project/vllm-ascend
80-
run-id: ${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}
102+
name: ${{ github.event.inputs.vllm-ascend-branch }}-${{ steps.get_basename.outputs.model_base_name }}-V${{ matrix.version }}-report
103+
path: ./docs/source/developer_guide/evaluation/accuracy_report_bak
104+
github-token: ${{ secrets.GITHUB_TOKEN }}
105+
repository: ${{ github.repository }}
106+
run-id: ${{ steps.get_run_id.outputs.runid }}
107+
108+
- name: Upload reports artifact
109+
if: ${{ steps.get_run_id.outputs.runid != '' }}
110+
uses: actions/upload-artifact@v4
111+
with:
112+
name: report-${{ steps.get_basename.outputs.model_base_name }}-v${{ matrix.version }}
113+
path: ./docs/source/developer_guide/evaluation/accuracy_report_bak/*.md
114+
retention-days: 90
81115

82-
- name: Download Qwen/Qwen2.5-7B-Instruct Artifact
83-
uses: actions/download-artifact@v4
116+
create_pr:
117+
runs-on: ubuntu-latest
118+
needs: download_reports
119+
steps:
120+
- name: Checkout repository
121+
uses: actions/checkout@v4
84122
with:
85-
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-7B-Instruct-V0-report
86-
path: ./docs/source/developer_guide/evaluation/accuracy_report
87-
github-token: ${{ secrets.GITHUB_TOKEN }}
88-
repository: vllm-project/vllm-ascend
89-
run-id: ${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
123+
ref: ${{ github.event.inputs.vllm-ascend-branch }}
124+
125+
- name: Setup workspace
126+
run: mkdir -p ./accuracy/accuracy_report
90127

91-
- name: Download Qwen/Qwen3-8B-Base Artifact
128+
- name: Download only current run reports
92129
uses: actions/download-artifact@v4
93130
with:
94-
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen3-8B-Base-V0-report
95131
path: ./docs/source/developer_guide/evaluation/accuracy_report
132+
pattern: report-*
96133
github-token: ${{ secrets.GITHUB_TOKEN }}
97-
repository: vllm-project/vllm-ascend
98-
run-id: ${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}
134+
run-id: ${{ github.run_id }}
135+
136+
- name: Delete old report
137+
run: |
138+
find ./docs/source/developer_guide/evaluation/accuracy_report -maxdepth 1 -type f -name '*.md' ! -name 'index.md' -delete
139+
find ./docs/source/developer_guide/evaluation/accuracy_report -mindepth 2 -type f -name '*.md' -exec mv -f {} ./docs/source/developer_guide/evaluation/accuracy_report \;
140+
find ./docs/source/developer_guide/evaluation/accuracy_report -mindepth 1 -type d -empty -delete
99141
100-
- name: Display Files
101-
working-directory: ./docs/source/developer_guide/evaluation/accuracy_report
142+
- name: Generate step summary
143+
if: ${{ always() }}
102144
run: |
103-
cat ./Qwen2.5-VL-7B-Instruct.md
104-
cat ./Qwen2.5-7B-Instruct.md
105-
cat ./Qwen3-8B-Base.md
106-
107-
- name: Create Pull Request for markdown update
145+
for report in ./docs/source/developer_guide/evaluation/accuracy_report/*.md; do
146+
filename=$(basename "$report")
147+
# skip index.md
148+
if [ "$filename" = "index.md" ]; then
149+
continue
150+
fi
151+
152+
if [ -f "$report" ]; then
153+
{
154+
echo -e "\n\n---\n"
155+
echo "## 📄 Report File: $(basename $report)"
156+
cat "$report"
157+
} >> "$GITHUB_STEP_SUMMARY"
158+
fi
159+
done
160+
161+
- name: Update accuracy_report/index.md
162+
run: |
163+
REPORT_DIR="./docs/source/developer_guide/evaluation/accuracy_report"
164+
INDEX_MD="$REPORT_DIR/index.md"
165+
166+
{
167+
echo "# Accuracy Report"
168+
echo ""
169+
echo "::: {toctree}"
170+
echo ":caption: Accuracy Report"
171+
echo ":maxdepth: 1"
172+
173+
for report in "$REPORT_DIR"/*.md; do
174+
filename="$(basename "$report" .md)"
175+
if [ "$filename" != "index" ]; then
176+
echo "$filename"
177+
fi
178+
done
179+
180+
echo ":::"
181+
} > "$INDEX_MD"
182+
183+
- name: Create Pull Request
108184
uses: peter-evans/create-pull-request@v7
109185
with:
110186
token: ${{ secrets.PR_TOKEN }}
111-
base: ${{ github.event.inputs.branch }}
112-
branch: auto-pr/accuracy-test
113-
commit-message: "Update accuracy report for ${{ github.event.inputs.branch }}"
187+
base: ${{ github.event.inputs.vllm-ascend-branch }}
188+
branch: auto-pr/accuracy-report
189+
commit-message: "Update accuracy reports for ${{ github.event.inputs.vllm-ascend-branch }}"
114190
add-paths: ./docs/source/developer_guide/evaluation/accuracy_report/*.md
115-
title: "[Doc]Update accuracy report for ${{ github.event.inputs.branch }}"
191+
title: "[Doc] Update accuracy reports for ${{ github.event.inputs.vllm-ascend-branch }}"
116192
body: |
117-
The accuracy results running on Ascend NPU have changed, I'm updating the report.
118-
Please review the changes.
119-
193+
The accuracy results running on NPU Altlas A2 have changed, updating reports for:
194+
${{
195+
github.event.inputs.models == 'all'
196+
&& 'All models (Qwen2.5-7B-Instruct, Qwen2.5-VL-7B-Instruct, Qwen3-8B-Base)'
197+
|| github.event.inputs.models
198+
}}
199+
120200
- [Workflow run][1]
121-
- [Qwen2.5-7B-Instruct accuracy report][2]
122-
- [Qwen2.5-VL-7B-Instruct accuracy report][3]
123-
- [Qwen3-8B-Base accuracy report][4]
124-
125-
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
126-
[2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
127-
[3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}
128-
[4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen3_8B_Base_latest_run_id_V0.outputs.runid }}
201+
202+
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

‎.github/workflows/accuracy_test.yaml

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ on:
3434
# Current supported vLLM versions
3535
options:
3636
- main
37-
- v0.9.0.1
38-
- v0.9.0
37+
- v0.9.1
3938
- v0.7.3
4039
vllm-ascend-version:
4140
description: 'vllm-ascend version:'
@@ -96,7 +95,7 @@ jobs:
9695
# - vl-accuracy-test: Qwen/Qwen2.5-VL-7B-Instruct
9796
model_name: ${{ fromJSON(
9897
(github.event.inputs.models == 'all' &&
99-
'["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","model_name":"Qwen/Qwen3-8B-Base"]') ||
98+
'["Qwen/Qwen2.5-7B-Instruct","Qwen/Qwen2.5-VL-7B-Instruct","Qwen/Qwen3-8B-Base"]') ||
10099
(github.event.inputs.models == 'Qwen/Qwen2.5-7B-Instruct' &&
101100
'["Qwen/Qwen2.5-7B-Instruct"]') ||
102101
(github.event.inputs.models == 'Qwen/Qwen2.5-VL-7B-Instruct' &&
@@ -159,7 +158,7 @@ jobs:
159158
repository: vllm-project/vllm
160159
path: ./vllm-empty
161160
# Please also update this when bump matched version
162-
ref: ${{ github.event.inputs.vllm-version || 'v0.9.0' }}
161+
ref: ${{ github.event.inputs.vllm-version || 'v0.9.1' }}
163162

164163
- name: Install vllm-project/vllm from source
165164
working-directory: ./vllm-empty
@@ -201,6 +200,7 @@ jobs:
201200
pip show torch | grep "Version:" | awk '{print "GHA_TORCH_VERSION="$2}'
202201
pip show torch_npu | grep "Version:" | awk '{print "GHA_TORCH_NPU_VERSION="$2}'
203202
pip show vllm | grep "Version:" | awk '{print "GHA_VLLM_VERSION="$2}' | sed 's/+.*//'
203+
echo "GHA_VLLM_ASCEND_VERSION=${{ github.event.inputs.vllm-ascend-version || github.ref }}"
204204
} >> "$GITHUB_ENV"
205205
206206
- name: Print versions
@@ -209,7 +209,7 @@ jobs:
209209
echo "Torch NPU: ${{ env.GHA_TORCH_NPU_VERSION }}"
210210
echo "Torch: ${{ env.GHA_TORCH_VERSION }}"
211211
echo "vLLM: ${{ env.GHA_VLLM_VERSION }}"
212-
echo "vLLM Ascend: ${{ env.GHA_VLLM_ASCEND_VERSION || github.ref }}"
212+
echo "vLLM Ascend: ${{ env.GHA_VLLM_ASCEND_VERSION }}"
213213
214214
- name: Run Accuracy Test for V${{ matrix.vllm_use_version }}
215215
id: report
@@ -238,10 +238,16 @@ jobs:
238238
run: |
239239
cat ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md >> $GITHUB_STEP_SUMMARY
240240
241+
- name: Sanitize version string for artifact naming
242+
run: |
243+
SAFE_VLLM_ASCEND_VERSION="${GHA_VLLM_ASCEND_VERSION//\//-}"
244+
echo "SAFE_VLLM_ASCEND_VERSION=$SAFE_VLLM_ASCEND_VERSION" >> "$GITHUB_ENV"
245+
241246
- name: Upload Report for V${{ matrix.vllm_use_version }}
247+
if: ${{ github.event_name == 'workflow_dispatch' }}
242248
uses: actions/upload-artifact@v4
243249
with:
244-
name: "${{ env.GHA_VLLM_ASCEND_VERSION }}-${{ steps.report.outputs.markdown_name }}-report"
250+
name: "${{ env.SAFE_VLLM_ASCEND_VERSION }}-${{ steps.report.outputs.markdown_name }}-report"
245251
path: ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md
246252
if-no-files-found: warn
247253
retention-days: 90

‎.github/workflows/actionlint.yml

Lines changed: 0 additions & 53 deletions
This file was deleted.

0 commit comments

Comments
 (0)