Skip to content

Commit def3863

Browse files
arjunsuresharjunsuresh
authored
Merge pull request #173 from mlcommons/dev
Sync Dev
2 parents 76e275c + 02683cf commit def3863

File tree

101 files changed

+945
-1549
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+945
-1549
lines changed

‎.github/workflows/build_wheel.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ on:
55
types: [published]
66
push:
77
branches:
8-
- main
98
- dev
109
paths:
1110
- VERSION

‎.github/workflows/test-amd-mlperf-inference-implementations.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
python3 -m venv gh_action
2121
source gh_action/bin/activate
2222
export MLC_REPOS=$HOME/GH_MLC
23-
pip install --upgrade cm4mlops
24-
cm pull repo
23+
pip install --upgrade mlc-scripts
24+
mlc pull repo
2525
mlcr --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=amd --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_mlc_repo=gateoverflow@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=rocm --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet --docker_skip_run_cmd=yes
2626
# mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=dev --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c

‎.github/workflows/test-intel-mlperf-inference-implementations.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
python3 -m venv gh_action_conda
2121
source gh_action_conda/bin/activate
2222
export MLC_REPOS=$HOME/GH_MLC
23-
pip install --upgrade cm4mlops
23+
pip install --upgrade mlc-scripts
2424
pip install tabulate
2525
mlcr --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c --implementation=intel --backend=pytorch --category=datacenter --division=open --scenario=Offline --docker_dt=yes --docker_it=no --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --device=cpu --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean --docker --quiet
2626
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_unofficial_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from GH action on SPR.24c" --quiet --submission_dir=$HOME/gh_action_submissions --hw_name=IntelSPR.24c

‎.github/workflows/test-mlc-script-features.yml

Lines changed: 83 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: MLC script automation features test
22

33
on:
4-
pull_request:
4+
pull_request_target:
55
branches: [ "main", "dev" ]
66
paths:
77
- '.github/workflows/test-mlc-script-features.yml'
@@ -61,23 +61,101 @@ jobs:
6161
mlcr --tags=python,src,install,_shared --version=3.9.10 --quiet
6262
mlc search cache --tags=python,src,install,_shared,version-3.9.10
6363
64+
test_docker:
65+
runs-on: ubuntu-latest
66+
strategy:
67+
fail-fast: false
68+
matrix:
69+
python-version: ["3.12", "3.8"]
70+
71+
steps:
72+
- uses: actions/checkout@v4
73+
- name: Set up Python ${{ matrix.python-version }}
74+
uses: actions/setup-python@v3
75+
with:
76+
python-version: ${{ matrix.python-version }}
77+
- name: Pull MLOps repository
78+
run: |
79+
pip install mlcflow
80+
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
81+
6482
- name: Run docker container from dockerhub on linux
65-
if: runner.os == 'linux'
6683
run: |
6784
mlcr --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=cm-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=cknowledge --quiet
6885
6986
- name: Run docker container locally on linux
70-
if: runner.os == 'linux'
7187
run: |
7288
mlcr --tags=run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet
7389
90+
test_mlperf_retinanet_cpp_venv:
91+
runs-on: ubuntu-latest
92+
strategy:
93+
fail-fast: false
94+
matrix:
95+
python-version: ["3.12", "3.8"]
96+
97+
steps:
98+
- uses: actions/checkout@v4
99+
- name: Set up Python ${{ matrix.python-version }}
100+
uses: actions/setup-python@v3
101+
with:
102+
python-version: ${{ matrix.python-version }}
103+
- name: Pull MLOps repository
104+
run: |
105+
pip install mlcflow
106+
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
107+
74108
- name: Run MLPerf Inference Retinanet with native and virtual Python
75-
if: runner.os == 'linux'
76109
run: |
77110
mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=accuracy --test_query_count=10 --rerun --quiet
78111
79112
mlcr --tags=app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --scenario=Offline --mode=performance --test_query_count=10 --rerun --quiet
80113
81114
mlcr --tags=install,python-venv --version=3.10.8 --name=mlperf --quiet
82115
83-
mlcr --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=Community --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
116+
mlcr --tags=run,mlperf,inference,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --adr.compiler.tags=gcc --adr.openimages-preprocessed.tags=_50 --submitter=MLCommons --implementation=cpp --hw_name=default --model=retinanet --backend=onnxruntime --device=cpu --scenario=Offline --quiet
117+
118+
# Step for Linux/MacOS
119+
- name: Randomly Execute Step (Linux/MacOS)
120+
if: runner.os != 'Windows'
121+
run: |
122+
RANDOM_NUMBER=$((RANDOM % 10))
123+
echo "Random number is $RANDOM_NUMBER"
124+
if [ "$RANDOM_NUMBER" -eq 0 ]; then
125+
echo "run_step=true" >> $GITHUB_ENV
126+
else
127+
echo "run_step=false" >> $GITHUB_ENV
128+
fi
129+
130+
# Step for Windows
131+
- name: Randomly Execute Step (Windows)
132+
if: runner.os == 'Windows'
133+
run: |
134+
$RANDOM_NUMBER = Get-Random -Maximum 10
135+
Write-Host "Random number is $RANDOM_NUMBER"
136+
if ($RANDOM_NUMBER -eq 0) {
137+
Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append
138+
} else {
139+
Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append
140+
}
141+
142+
- name: Retrieve secrets from Keeper
143+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
144+
id: ksecrets
145+
uses: Keeper-Security/ksm-action@master
146+
with:
147+
keeper-secret-config: ${{ secrets.KSM_CONFIG }}
148+
secrets: |-
149+
ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT
150+
- name: Push Results
151+
env:
152+
GITHUB_TOKEN: ${{ env.PAT }}
153+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
154+
run: |
155+
git config --global user.name "mlcommons-bot"
156+
git config --global user.email "mlcommons-bot@users.noreply.github.com"
157+
git config --global credential.https://github.com.helper ""
158+
git config --global credential.https://github.com.helper "!gh auth git-credential"
159+
git config --global credential.https://gist.github.com.helper ""
160+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
161+
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

‎.github/workflows/test-mlperf-wheel.yml renamed to ‎.github/workflows/test-mlcscripts-wheel.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
name: Build Python Wheel
1+
name: Build mlc-scripts Wheel
22

33
on:
44
pull_request:
55
branches:
66
- main
77
- dev
88
paths:
9-
- '.github/workflows/test-mlperf-wheel.yml'
9+
- '.github/workflows/test-mlcscripts-wheel.yml'
1010
- 'setup.py'
1111

1212
jobs:
@@ -16,6 +16,9 @@ jobs:
1616
matrix:
1717
os: [macos-latest, ubuntu-latest, windows-latest]
1818
python-version: [ '3.8', '3.13']
19+
exclude:
20+
- os: windows-latest
21+
python-version: "3.8"
1922

2023
runs-on: ${{ matrix.os }}
2124

‎.github/workflows/test-mlperf-inference-gptj.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ jobs:
2424
source gh_action/bin/deactivate || python3 -m venv gh_action
2525
source gh_action/bin/activate
2626
export MLC_REPOS=$HOME/GH_MLC
27-
python3 -m pip install cm4mlops
28-
cm pull repo
27+
python3 -m pip install --upgrade mlc-scripts
28+
mlc pull repo
2929
mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --pull_changes=yes --pull_inference_changes=yes --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_mlc_repo=gateoverflow@mlperf-automations --docker_mlc_repo_branch=dev --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --get_platform_details=yes --implementation=reference --clean
3030
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
3131

‎.github/workflows/test-mlperf-inference-llama2.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ jobs:
2525
source gh_action/bin/deactivate || python3 -m venv gh_action
2626
source gh_action/bin/activate
2727
export MLC_REPOS=$HOME/GH_MLC
28-
pip install cm4mlops
28+
pip install mlc-scripts
2929
pip install tabulate
30-
cm pull repo
30+
mlc pull repo
3131
pip install "huggingface_hub[cli]"
3232
git config --global credential.helper store
3333
huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential

‎.github/workflows/test-mlperf-inference-mixtral.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ jobs:
2626
source gh_action/bin/deactivate || python3 -m venv gh_action
2727
source gh_action/bin/activate
2828
export MLC_REPOS=$HOME/GH_MLC
29-
pip install cm4mlops
29+
pip install --upgrade mlc-scripts
3030
pip install "huggingface_hub[cli]"
3131
git config --global credential.helper store
3232
huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
33-
cm pull repo
33+
mlc pull repo
3434
mlcr --tags=run-mlperf,inference,_submission,_short --adr.inference-src.tags=_branch.dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_mlc_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=3 --target_qps=0.001 --clean --env.MLC_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.MLC_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --adr.openorca-mbxp-gsm8k-combined-preprocessed.tags=_size.1
3535
mlcr --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

‎.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
1-
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2-
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3-
41
name: MLPerf inference MLCommons C++ ResNet50
52

63
on:
7-
pull_request:
8-
branches: [ "main", "dev", "mlperf-inference" ]
4+
pull_request_target:
5+
branches: [ "main", "dev" ]
96
paths:
107
- '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml'
118
- '**'

‎.github/workflows/test-mlperf-inference-resnet50.yml

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,16 +58,30 @@ jobs:
5858
if: matrix.os != 'windows-latest'
5959
run: |
6060
mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
61-
- name: Randomly Execute Step
62-
id: random-check
61+
# Step for Linux/MacOS
62+
- name: Randomly Execute Step (Linux/MacOS)
63+
if: runner.os != 'Windows'
6364
run: |
64-
RANDOM_NUMBER=$((RANDOM % 10))
65-
echo "Random number is $RANDOM_NUMBER"
66-
if [ "$RANDOM_NUMBER" -eq 0 ]; then
67-
echo "run_step=true" >> $GITHUB_ENV
68-
else
69-
echo "run_step=false" >> $GITHUB_ENV
70-
fi
65+
RANDOM_NUMBER=$((RANDOM % 10))
66+
echo "Random number is $RANDOM_NUMBER"
67+
if [ "$RANDOM_NUMBER" -eq 0 ]; then
68+
echo "run_step=true" >> $GITHUB_ENV
69+
else
70+
echo "run_step=false" >> $GITHUB_ENV
71+
fi
72+
73+
# Step for Windows
74+
- name: Randomly Execute Step (Windows)
75+
if: runner.os == 'Windows'
76+
run: |
77+
$RANDOM_NUMBER = Get-Random -Maximum 10
78+
Write-Host "Random number is $RANDOM_NUMBER"
79+
if ($RANDOM_NUMBER -eq 0) {
80+
Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append
81+
} else {
82+
Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append
83+
}
84+
7185
- name: Retrieve secrets from Keeper
7286
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
7387
id: ksecrets

0 commit comments

Comments
 (0)