Skip to content

Commit 7b0d537

Browse files
arjunsuresharjunsuresh
authored
Update github tests to use mlc repo, mlc-scripts package name added
* Update test-mlperf-inference-resnet50.yml * Support nvidia implementation for v5.0 * Update and rename test-mlc-tutorial-tvm.yml to test-mlperf-inference-tvm-resnet50.yml * Update test-mlc-script-features.yml | Added retinanet test * Delete .github/workflows/test-mlc-tutorial-retinanet.yml * Use mlc-scripts name
1 parent e91a7b2 commit 7b0d537

File tree

11 files changed

+73
-122
lines changed

11 files changed

+73
-122
lines changed

‎.github/workflows/test-mlc-script-features.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,12 @@ jobs:
7070
run: |
7171
mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet
7272
73+
- name: Run MLPerf Inference Retinanet with native and virtual Python
74+
run: |
75+
mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=accuracy --test_query_count=10 --rerun --quiet
76+
77+
mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=performance --test_query_count=10 --rerun --quiet
78+
79+
mlcr --tags=install,python-venv --version=3.10.8 --name=mlperf --quiet
80+
81+
mlcr --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=Community --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet

‎.github/workflows/test-mlc-tutorial-retinanet.yml

Lines changed: 0 additions & 42 deletions
This file was deleted.

‎.github/workflows/test-mlperf-inference-resnet50.yml

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ jobs:
2727
backend: tf
2828
- os: windows-latest
2929
implementation: cpp
30+
3031
steps:
3132
- uses: actions/checkout@v4
3233
with:
@@ -56,8 +57,9 @@ jobs:
5657
- name: Test MLPerf Inference ResNet50 (Linux/macOS)
5758
if: matrix.os != 'windows-latest'
5859
run: |
59-
mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
60+
mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
6061
- name: Retrieve secrets from Keeper
62+
if: github.repository_owner == 'mlcommons'
6163
id: ksecrets
6264
uses: Keeper-Security/ksm-action@master
6365
with:
@@ -67,6 +69,7 @@ jobs:
6769
- name: Push Results
6870
env:
6971
GITHUB_TOKEN: ${{ env.PAT }}
72+
if: github.repository_owner == 'mlcommons'
7073
run: |
7174
git config --global user.name "mlcommons-bot"
7275
git config --global user.email "mlcommons-bot@users.noreply.github.com"
@@ -75,3 +78,15 @@ jobs:
7578
git config --global credential.https://gist.github.com.helper ""
7679
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
7780
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
81+
- name: Push Results
82+
env:
83+
GITHUB_TOKEN: ${{ secrets.PAT1 }}
84+
if: github.repository_owner == 'gateoverflow'
85+
run: |
86+
git config --global user.name "mlcommons-bot"
87+
git config --global user.email "mlcommons-bot@users.noreply.github.com"
88+
git config --global credential.https://github.com.helper ""
89+
git config --global credential.https://github.com.helper "!gh auth git-credential"
90+
git config --global credential.https://gist.github.com.helper ""
91+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
92+
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

‎.github/workflows/test-mlc-tutorial-tvm.yml renamed to ‎.github/workflows/test-mlperf-inference-tvm-resnet50.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
1-
name: MLC tutorial tvm
1+
name: MLPerf Inference TVM ResNet50
22

33
on:
44
pull_request_target:
55
branches: [ "main", "dev" ]
66
paths:
7-
- '.github/workflows/test-mlc-tutorial-tvm.yml'
7+
- '.github/workflows/test-mlperf-inference-tvm-resnet50.yml'
88
- '**'
99
- '!**.md'
1010

1111
jobs:
12-
build:
13-
12+
tvm-run:
1413
runs-on: ubuntu-latest
1514
strategy:
1615
fail-fast: false

‎.github/workflows/test-mlperf-inference-tvm.yml

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎pyproject.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0", "wheel", "toml", "mlcflow"]
33
build-backend = "setuptools.build_meta"
44

55
[project]
6-
name = "mlperf"
6+
name = "mlc-scripts"
77
version = "0.0.1"
88
description = "Automation scripts for running ML applications using MLC interface"
99
authors = [
@@ -12,7 +12,7 @@ authors = [
1212
license = { file = "LICENSE.md" }
1313
readme = "README.md"
1414
requires-python = ">=3.7"
15-
keywords = ["mlc", "mlcflow", "pypi", "package", "automation", "mlperf", "mlcr"]
15+
keywords = ["mlc", "mlcscripts", "mlcflow", "pypi", "package", "automation", "mlperf", "mlcr"]
1616
classifiers = [
1717
"Programming Language :: Python :: 3",
1818
"License :: OSI Approved :: Apache Software License",

‎script/app-mlperf-inference/meta.yaml

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,27 @@ variations:
356356
env:
357357
MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp310-cp310-linux_aarch64.whl'
358358

359+
nvidia-original,r5.0-dev_default:
360+
docker:
361+
build_deps:
362+
- tags: detect,os
363+
image_name: mlperf-inference-nvidia-v5.0-dev-common
364+
update_meta_if_env:
365+
- enable_if_env:
366+
MLC_HOST_PLATFORM_FLAVOR:
367+
- x86_64
368+
docker:
369+
base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-x86_64-ubuntu20.04-public
370+
env:
371+
MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp38-cp38-linux_x86_64.whl'
372+
373+
- skip_if_env:
374+
MLC_HOST_PLATFORM_FLAVOR:
375+
- x86_64
376+
docker:
377+
base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v4.0-cuda12.2-cudnn8.9-aarch64-ubuntu22.04-public
378+
env:
379+
MLC_ENV_NVMITTEN_DOCKER_WHEEL_PATH: '/opt/nvmitten-0.1.3b0-cp310-cp310-linux_aarch64.whl'
359380

360381

361382
nvidia-original,gptj_:
@@ -1716,10 +1737,10 @@ variations:
17161737
reproducibility
17171738
add_deps_recursive:
17181739
nvidia-inference-common-code:
1719-
version: r4.1
1740+
version: r4.1-dev
17201741
tags: _mlcommons
17211742
nvidia-inference-server:
1722-
version: r4.1
1743+
version: r4.0
17231744
tags: _mlcommons
17241745
intel-harness:
17251746
tags: _v4.1

‎script/build-mlperf-inference-server-nvidia/meta.yaml

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -228,25 +228,9 @@ variations:
228228
- 8
229229

230230
versions:
231-
r2.1:
232-
add_deps_recursive:
233-
nvidia-inference-common-code:
234-
version: r2.1
235-
nvidia-scratch-space:
236-
tags: _version.2_1
237-
238-
r3.0:
239-
add_deps_recursive:
240-
nvidia-inference-common-code:
241-
version: r3.0
242-
nvidia-scratch-space:
243-
tags: _version.3_0
231+
r2.1: {}
232+
r3.0: {}
244233
r3.1:
245-
add_deps_recursive:
246-
nvidia-inference-common-code:
247-
version: r3.1
248-
nvidia-scratch-space:
249-
tags: _version.4_0
250234
deps:
251235
- tags: install,pytorch,from.src,_for-nvidia-mlperf-inference-v3.1
252236
names:
@@ -259,11 +243,6 @@ versions:
259243
- tags: install,nccl,libs,_cuda
260244

261245
r4.0:
262-
add_deps_recursive:
263-
nvidia-inference-common-code:
264-
version: r4.0
265-
nvidia-scratch-space:
266-
tags: _version.4_0
267246
default_env:
268247
BUILD_TRTLLM: 1
269248
deps:
@@ -301,11 +280,6 @@ versions:
301280
- 8
302281

303282
r4.1-dev:
304-
add_deps_recursive:
305-
nvidia-inference-common-code:
306-
version: r4.0
307-
nvidia-scratch-space:
308-
tags: _version.4_1
309283
default_env:
310284
BUILD_TRTLLM: 1
311285
deps:
@@ -343,11 +317,6 @@ versions:
343317
- 8
344318

345319
r4.1:
346-
add_deps_recursive:
347-
nvidia-inference-common-code:
348-
version: r4.1
349-
nvidia-scratch-space:
350-
tags: _version.4_1
351320
default_env:
352321
BUILD_TRTLLM: 1
353322

‎script/get-mlperf-inference-nvidia-common-code/meta.yaml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ deps:
99
- inherit_variation_tags: true
1010
names:
1111
- mlperf-inference-results
12-
tags: get,mlperf,inference,results,official,_code-only
12+
tags: get,mlperf,inference,results,official
1313
new_env_keys:
1414
- +PYTHONPATH
1515
- MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH
@@ -37,19 +37,29 @@ versions:
3737
add_deps_recursive:
3838
mlperf-inference-results:
3939
version: v2.1
40+
tags: _code-only
4041
r3.0:
4142
add_deps_recursive:
4243
mlperf-inference-results:
4344
version: v3.0
45+
tags: _code-only
4446
r3.1:
4547
add_deps_recursive:
4648
mlperf-inference-results:
4749
version: v3.1
50+
tags: _code-only
4851
r4.0:
4952
add_deps_recursive:
5053
mlperf-inference-results:
5154
version: v4.0
55+
tags: _code-only
5256
r4.1:
5357
add_deps_recursive:
5458
mlperf-inference-results:
5559
version: v4.1
60+
tags: _code-only
61+
r4.1-dev:
62+
add_deps_recursive:
63+
mlperf-inference-results:
64+
version: v4.0
65+
tags: _code-only-for-v5.0

‎script/get-mlperf-inference-results/meta.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ variations:
3838
inference-results-repo:
3939
tags: _branch.mlc-code-only
4040
group: repo-branch
41+
code-only-for-v5.0:
42+
adr:
43+
inference-results-repo:
44+
tags: _branch.mlc-code-only-for-v5.0
45+
group: repo-branch
4146
ctuning:
4247
env:
4348
GITHUB_REPO_OWNER: ctuning

0 commit comments

Comments
 (0)