Skip to content

Commit 51a9f12

Browse files
anandhu-enganandhu-eng
authored
Merge pull request #289 from sujik18/dev
Added Resnet50 closed division github action
2 parents e023a0f + ea48b11 commit 51a9f12

File tree

1 file changed

+143
-0
lines changed

1 file changed

+143
-0
lines changed

‎.github/workflows/test-mlperf-inference-resnet50-closed-division.yml

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
name: MLPerf inference ResNet50 Closed Division for testing Compliance
2+
3+
on:
4+
# push:
5+
# branches:
6+
# - test
7+
schedule:
8+
- cron: '0 0 * * 0' # Runs once a week
9+
pull_request_target:
10+
branches: [ "main", "dev" ]
11+
paths:
12+
- '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml'
13+
- '**'
14+
- '!**.md'
15+
# workflow_dispatch: # Allows manual triggering of the workflow
16+
jobs:
17+
build:
18+
name: MLPerf inference MLCommons ResNet50 Closed Division
19+
runs-on: ${{ matrix.os }}
20+
env:
21+
MLC_INDEX: "on"
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
os: [ubuntu-latest, windows-latest, macos-latest]
26+
python-version: [ "3.12" ]
27+
backend: [ "onnxruntime", "tf" ]
28+
implementation: [ "python", "cpp" ]
29+
exclude:
30+
- backend: tf
31+
implementation: cpp
32+
- os: macos-latest
33+
backend: tf
34+
- os: windows-latest
35+
implementation: cpp
36+
steps:
37+
- uses: actions/checkout@v4
38+
with:
39+
fetch-depth: 0
40+
41+
- name: Set up Python ${{ matrix.python-version }}
42+
uses: actions/setup-python@v3
43+
with:
44+
python-version: ${{ matrix.python-version }}
45+
- name: Configure git longpaths (Windows)
46+
if: matrix.os == 'windows-latest'
47+
run: |
48+
git config --system core.longpaths true
49+
50+
- name: Install mlcflow
51+
run: |
52+
pip install mlcflow
53+
pip install tabulate
54+
55+
- name: Pull MLOps repo
56+
run: |
57+
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
58+
59+
60+
- name: Test MLPerf Inference ResNet50 Offline (Windows)
61+
if: matrix.os == 'windows-latest'
62+
run: |
63+
mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid
64+
65+
- name: Test MLPerf Inference ResNet50 Singlestream (Windows)
66+
if: matrix.os == 'windows-latest'
67+
run: |
68+
mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --target_latency=10
69+
70+
- name: Test MLPerf Inference ResNet50 Multistream (Windows)
71+
if: matrix.os == 'windows-latest'
72+
run: |
73+
mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream , --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10
74+
75+
- name: Compliance Test MLPerf Inference ResNet50 (Windows)
76+
if: matrix.os == 'windows-latest'
77+
run: mlcr run-mlperf, inference, _submission, _short,_all-scenarios, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes
78+
79+
- name: Test MLPerf Inference ResNet50 Offline(Linux/macOS)
80+
if: matrix.os != 'windows-latest'
81+
run: |
82+
mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid
83+
84+
- name: Test MLPerf Inference ResNet50 SingleStream(Linux/macOS)
85+
if: matrix.os != 'windows-latest'
86+
run: |
87+
mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10
88+
89+
- name: Test MLPerf Inference ResNet50 MultiStream(Linux/macOS)
90+
if: matrix.os != 'windows-latest'
91+
run: |
92+
mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10
93+
94+
- name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS)
95+
if: matrix.os != 'windows-latest'
96+
run: |
97+
mlcr run-mlperf, inference, _submission, _short, _all-scenarios --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 -v --quiet --execution_mode=valid --compliance=yes
98+
99+
# Step for Linux/MacOS
100+
- name: Randomly Execute Step (Linux/MacOS)
101+
if: runner.os != 'Windows'
102+
run: |
103+
RANDOM_NUMBER=$((RANDOM % 10))
104+
echo "Random number is $RANDOM_NUMBER"
105+
if [ "$RANDOM_NUMBER" -eq 0 ]; then
106+
echo "run_step=true" >> $GITHUB_ENV
107+
else
108+
echo "run_step=false" >> $GITHUB_ENV
109+
fi
110+
111+
# Step for Windows
112+
- name: Randomly Execute Step (Windows)
113+
if: runner.os == 'Windows'
114+
run: |
115+
$RANDOM_NUMBER = Get-Random -Maximum 10
116+
Write-Host "Random number is $RANDOM_NUMBER"
117+
if ($RANDOM_NUMBER -eq 0) {
118+
Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append
119+
} else {
120+
Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append
121+
}
122+
123+
- name: Retrieve secrets from Keeper
124+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
125+
id: ksecrets
126+
uses: Keeper-Security/ksm-action@master
127+
with:
128+
keeper-secret-config: ${{ secrets.KSM_CONFIG }}
129+
secrets: |-
130+
ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT
131+
- name: Push Results
132+
env:
133+
GITHUB_TOKEN: ${{ env.PAT }}
134+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
135+
run: |
136+
git config --global user.name "mlcommons-bot"
137+
git config --global user.email "mlcommons-bot@users.noreply.github.com"
138+
git config --global credential.https://github.com.helper ""
139+
git config --global credential.https://github.com.helper "!gh auth git-credential"
140+
git config --global credential.https://gist.github.com.helper ""
141+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
142+
mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
143+

0 commit comments

Comments
 (0)