diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml new file mode 100644 index 000000000..574d0ae19 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -0,0 +1,143 @@ +name: MLPerf inference ResNet50 Closed Division for testing Compliance + +on: + # push: + # branches: + # - test + schedule: + - cron: '0 0 * * 0' # Runs once a week + pull_request_target: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' + - '**' + - '!**.md' + # workflow_dispatch: # Allows manual triggering of the workflow +jobs: + build: + name: MLPerf inference MLCommons ResNet50 Closed Division + runs-on: ${{ matrix.os }} + env: + MLC_INDEX: "on" + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + backend: [ "onnxruntime", "tf" ] + implementation: [ "python", "cpp" ] + exclude: + - backend: tf + implementation: cpp + - os: macos-latest + backend: tf + - os: windows-latest + implementation: cpp + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Configure git longpaths (Windows) + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + + - name: Install mlcflow + run: | + pip install mlcflow + pip install tabulate + + - name: Pull MLOps repo + run: | + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + + - name: Test MLPerf Inference ResNet50 Offline (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + + - name: Test MLPerf Inference ResNet50 Singlestream (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --target_latency=10 + + - name: Test MLPerf Inference ResNet50 Multistream (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream , --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + + - name: Compliance Test MLPerf Inference ResNet50 (Windows) + if: matrix.os == 'windows-latest' + run: mlcr run-mlperf, inference, _submission, _short,_all-scenarios, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + + - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + + - name: Test MLPerf Inference ResNet50 SingleStream(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + + - name: Test MLPerf Inference ResNet50 MultiStream(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + + - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, _all-scenarios --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 -v --quiet --execution_mode=valid --compliance=yes + + # Step for Linux/MacOS + - name: Randomly Execute Step (Linux/MacOS) + if: runner.os != 'Windows' + run: | + RANDOM_NUMBER=$((RANDOM % 10)) + echo "Random number is $RANDOM_NUMBER" + if [ "$RANDOM_NUMBER" -eq 0 ]; then + echo "run_step=true" >> $GITHUB_ENV + else + echo "run_step=false" >> $GITHUB_ENV + fi + + # Step for Windows + - name: Randomly Execute Step (Windows) + if: runner.os == 'Windows' + run: | + $RANDOM_NUMBER = Get-Random -Maximum 10 + Write-Host "Random number is $RANDOM_NUMBER" + if ($RANDOM_NUMBER -eq 0) { + Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append + } else { + Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append + } + + - name: Retrieve secrets from Keeper + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + id: ksecrets + uses: Keeper-Security/ksm-action@master + with: + keeper-secret-config: ${{ secrets.KSM_CONFIG }} + secrets: |- + ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT + - name: Push Results + env: + GITHUB_TOKEN: ${{ env.PAT }} + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + run: | + git config --global user.name "mlcommons-bot" + git config --global user.email "mlcommons-bot@users.noreply.github.com" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet +