From 1fb2579c86d2d212f346766e3d4cc74ca4713ec5 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Wed, 5 Mar 2025 12:10:27 +0530 Subject: [PATCH 1/4] Added Resnet50 closed division github action Fix for #247 --- ...erf-inference-resnet50-closed-division.yml | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-resnet50-closed-division.yml diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml new file mode 100644 index 000000000..0822b7f37 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -0,0 +1,133 @@ +name: MLPerf inference ResNet50 Closed Division for testing Compliance + +on: + pull_request_target: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' + - '**' + - '!**.md' +jobs: + build: + name: MLPerf inference MLCommons ResNet50 Closed Division + runs-on: ${{ matrix.os }} + env: + MLC_INDEX: "on" + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + backend: [ "onnxruntime", "tf" ] + implementation: [ "python", "cpp" ] + exclude: + - backend: tf + implementation: cpp + - os: macos-latest + backend: tf + - os: windows-latest + implementation: cpp + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if the submission is for closed division + run: | + if git diff --name-only origin/${{ github.base_ref }}..HEAD | xargs grep -l "division=closed"; then + echo "DIVISION_CLOSED=true" >> $GITHUB_ENV + else + echo "DIVISION_CLOSED=false" >> $GITHUB_ENV + fi + + - name: Check value for the divison + run: echo "DIVISION_CLOSED=${{ env.DIVISION_CLOSED }}" + + - name: Skip workflow if its an open division submission as no compliance test is required + if: ${{ env.DIVISION_CLOSED == 'false' }} + run: | + echo "Skipping workflow because division=closed is not found in the changes." + exit 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Configure git longpaths (Windows) + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + + - name: Install mlcflow + run: | + pip install mlcflow + pip install tabulate + + - name: Pull MLOps repo + run: | + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + - name: Test MLPerf Inference ResNet50 (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet + + - name: Compliance Test MLPerf Inference ResNet50 (Windows) + if: matrix.os == 'windows-latest' + run: mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + + - name: Test MLPerf Inference ResNet50 (Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + + - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes + + + # Step for Linux/MacOS + - name: Randomly Execute Step (Linux/MacOS) + if: runner.os != 'Windows' + run: | + RANDOM_NUMBER=$((RANDOM % 10)) + echo "Random number is $RANDOM_NUMBER" + if [ "$RANDOM_NUMBER" -eq 0 ]; then + echo "run_step=true" >> $GITHUB_ENV + else + echo "run_step=false" >> $GITHUB_ENV + fi + + # Step for Windows + - name: Randomly Execute Step (Windows) + if: runner.os == 'Windows' + run: | + $RANDOM_NUMBER = Get-Random -Maximum 10 + Write-Host "Random number is $RANDOM_NUMBER" + if ($RANDOM_NUMBER -eq 0) { + Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append + } else { + Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append + } + + - name: Retrieve secrets from Keeper + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + id: ksecrets + uses: Keeper-Security/ksm-action@master + with: + keeper-secret-config: ${{ secrets.KSM_CONFIG }} + secrets: |- + ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT + - name: Push Results + env: + GITHUB_TOKEN: ${{ env.PAT }} + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + run: | + git config --global user.name "mlcommons-bot" + git config --global user.email "mlcommons-bot@users.noreply.github.com" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet + From 234aad7aa480cbd63d06c0234ef2dc0a353078d8 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Wed, 5 Mar 2025 12:22:36 +0530 Subject: [PATCH 2/4] Scheduling the workflow once a week --- .../test-mlperf-inference-resnet50-closed-division.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 0822b7f37..9c98f773c 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -1,6 +1,8 @@ name: MLPerf inference ResNet50 Closed Division for testing Compliance on: + schedule: + - cron: '0 0 * * 0' # Runs once a week pull_request_target: branches: [ "main", "dev" ] paths: From beda189cf1f1058454fe3a1c2a0242478ddb0386 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Thu, 6 Mar 2025 07:12:21 +0530 Subject: [PATCH 3/4] Changed the dataset parameter to short --- .../test-mlperf-inference-resnet50-closed-division.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 9c98f773c..1798ac9e8 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -9,6 +9,7 @@ on: - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - '**' - '!**.md' + workflow_dispatch: # Allows manual triggering of the workflow jobs: build: name: MLPerf inference MLCommons ResNet50 Closed Division @@ -72,20 +73,20 @@ jobs: - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet - name: Compliance Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' - run: mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + run: mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes - name: Test MLPerf Inference ResNet50 (Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes # Step for Linux/MacOS From ea48b11b8819352d36e97252d56d4f068128fa74 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Fri, 7 Mar 2025 10:03:01 +0530 Subject: [PATCH 4/4] Fixes for test-mlpref-inference-resnet50-closed-division.yml --- ...erf-inference-resnet50-closed-division.yml | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 1798ac9e8..574d0ae19 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -1,6 +1,9 @@ name: MLPerf inference ResNet50 Closed Division for testing Compliance on: + # push: + # branches: + # - test schedule: - cron: '0 0 * * 0' # Runs once a week pull_request_target: @@ -9,7 +12,7 @@ on: - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - '**' - '!**.md' - workflow_dispatch: # Allows manual triggering of the workflow + # workflow_dispatch: # Allows manual triggering of the workflow jobs: build: name: MLPerf inference MLCommons ResNet50 Closed Division @@ -35,23 +38,6 @@ jobs: with: fetch-depth: 0 - - name: Check if the submission is for closed division - run: | - if git diff --name-only origin/${{ github.base_ref }}..HEAD | xargs grep -l "division=closed"; then - echo "DIVISION_CLOSED=true" >> $GITHUB_ENV - else - echo "DIVISION_CLOSED=false" >> $GITHUB_ENV - fi - - - name: Check value for the divison - run: echo "DIVISION_CLOSED=${{ env.DIVISION_CLOSED }}" - - - name: Skip workflow if its an open division submission as no compliance test is required - if: ${{ env.DIVISION_CLOSED == 'false' }} - run: | - echo "Skipping workflow because division=closed is not found in the changes." - exit 0 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: @@ -69,25 +55,46 @@ jobs: - name: Pull MLOps repo run: | mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + + - name: Test MLPerf Inference ResNet50 Offline (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - name: Test MLPerf Inference ResNet50 (Windows) + - name: Test MLPerf Inference ResNet50 Singlestream (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet - + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --target_latency=10 + + - name: Test MLPerf Inference ResNet50 Multistream (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream , --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Compliance Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' - run: mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + run: mlcr run-mlperf, inference, _submission, _short,_all-scenarios, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + + - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - name: Test MLPerf Inference ResNet50 (Linux/macOS) + - name: Test MLPerf Inference ResNet50 SingleStream(Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Test MLPerf Inference ResNet50 MultiStream(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) + if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes - + mlcr run-mlperf, inference, _submission, _short, _all-scenarios --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 -v --quiet --execution_mode=valid --compliance=yes # Step for Linux/MacOS - name: Randomly Execute Step (Linux/MacOS)