Skip to content

Commit 1fb2579

Browse files
sujik18sujik18
committed
Added Resnet50 closed division github action
Fix for #247
1 parent 854c511 commit 1fb2579

File tree

1 file changed

+133
-0
lines changed

1 file changed

+133
-0
lines changed

‎.github/workflows/test-mlperf-inference-resnet50-closed-division.yml

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
name: MLPerf inference ResNet50 Closed Division for testing Compliance
2+
3+
on:
4+
pull_request_target:
5+
branches: [ "main", "dev" ]
6+
paths:
7+
- '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml'
8+
- '**'
9+
- '!**.md'
10+
jobs:
11+
build:
12+
name: MLPerf inference MLCommons ResNet50 Closed Division
13+
runs-on: ${{ matrix.os }}
14+
env:
15+
MLC_INDEX: "on"
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
os: [ubuntu-latest, windows-latest, macos-latest]
20+
python-version: [ "3.12" ]
21+
backend: [ "onnxruntime", "tf" ]
22+
implementation: [ "python", "cpp" ]
23+
exclude:
24+
- backend: tf
25+
implementation: cpp
26+
- os: macos-latest
27+
backend: tf
28+
- os: windows-latest
29+
implementation: cpp
30+
steps:
31+
- uses: actions/checkout@v4
32+
with:
33+
fetch-depth: 0
34+
35+
- name: Check if the submission is for closed division
36+
run: |
37+
if git diff --name-only origin/${{ github.base_ref }}..HEAD | xargs grep -l "division=closed"; then
38+
echo "DIVISION_CLOSED=true" >> $GITHUB_ENV
39+
else
40+
echo "DIVISION_CLOSED=false" >> $GITHUB_ENV
41+
fi
42+
43+
- name: Check value for the divison
44+
run: echo "DIVISION_CLOSED=${{ env.DIVISION_CLOSED }}"
45+
46+
- name: Skip workflow if its an open division submission as no compliance test is required
47+
if: ${{ env.DIVISION_CLOSED == 'false' }}
48+
run: |
49+
echo "Skipping workflow because division=closed is not found in the changes."
50+
exit 0
51+
52+
- name: Set up Python ${{ matrix.python-version }}
53+
uses: actions/setup-python@v3
54+
with:
55+
python-version: ${{ matrix.python-version }}
56+
- name: Configure git longpaths (Windows)
57+
if: matrix.os == 'windows-latest'
58+
run: |
59+
git config --system core.longpaths true
60+
61+
- name: Install mlcflow
62+
run: |
63+
pip install mlcflow
64+
pip install tabulate
65+
66+
- name: Pull MLOps repo
67+
run: |
68+
mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
69+
70+
- name: Test MLPerf Inference ResNet50 (Windows)
71+
if: matrix.os == 'windows-latest'
72+
run: |
73+
mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet
74+
75+
- name: Compliance Test MLPerf Inference ResNet50 (Windows)
76+
if: matrix.os == 'windows-latest'
77+
run: mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes
78+
79+
- name: Test MLPerf Inference ResNet50 (Linux/macOS)
80+
if: matrix.os != 'windows-latest'
81+
run: |
82+
mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet
83+
84+
- name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS)
85+
run: |
86+
mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes
87+
88+
89+
# Step for Linux/MacOS
90+
- name: Randomly Execute Step (Linux/MacOS)
91+
if: runner.os != 'Windows'
92+
run: |
93+
RANDOM_NUMBER=$((RANDOM % 10))
94+
echo "Random number is $RANDOM_NUMBER"
95+
if [ "$RANDOM_NUMBER" -eq 0 ]; then
96+
echo "run_step=true" >> $GITHUB_ENV
97+
else
98+
echo "run_step=false" >> $GITHUB_ENV
99+
fi
100+
101+
# Step for Windows
102+
- name: Randomly Execute Step (Windows)
103+
if: runner.os == 'Windows'
104+
run: |
105+
$RANDOM_NUMBER = Get-Random -Maximum 10
106+
Write-Host "Random number is $RANDOM_NUMBER"
107+
if ($RANDOM_NUMBER -eq 0) {
108+
Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append
109+
} else {
110+
Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append
111+
}
112+
113+
- name: Retrieve secrets from Keeper
114+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
115+
id: ksecrets
116+
uses: Keeper-Security/ksm-action@master
117+
with:
118+
keeper-secret-config: ${{ secrets.KSM_CONFIG }}
119+
secrets: |-
120+
ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT
121+
- name: Push Results
122+
env:
123+
GITHUB_TOKEN: ${{ env.PAT }}
124+
if: github.repository_owner == 'mlcommons' && env.run_step == 'true'
125+
run: |
126+
git config --global user.name "mlcommons-bot"
127+
git config --global user.email "mlcommons-bot@users.noreply.github.com"
128+
git config --global credential.https://github.com.helper ""
129+
git config --global credential.https://github.com.helper "!gh auth git-credential"
130+
git config --global credential.https://gist.github.com.helper ""
131+
git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
132+
mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
133+

0 commit comments

Comments
 (0)