Update test-mlperf-inference-abtf-poc.yml

.github/scripts/process_individual_tests.py

@@ -25,10 +25,10 @@
     ii = {
         'action': 'test', 'target': 'script', 'item': uid, 'quiet': 'yes', 'out': 'con'
     }
-    if os.environ.get('DOCKER_CM_REPO', '') != '':
-        ii['docker_cm_repo'] = os.environ['DOCKER_CM_REPO']
-    if os.environ.get('DOCKER_CM_REPO_BRANCH', '') != '':
-        ii['docker_cm_repo_branch'] = os.environ['DOCKER_CM_REPO_BRANCH']
+    if os.environ.get('DOCKER_MLC_REPO', '') != '':
+        ii['docker_cm_repo'] = os.environ['DOCKER_MLC_REPO']
+    if os.environ.get('DOCKER_MLC_REPO_BRANCH', '') != '':
+        ii['docker_cm_repo_branch'] = os.environ['DOCKER_MLC_REPO_BRANCH']
     if os.environ.get('TEST_INPUT_INDEX', '') != '':
         ii['test_input_index'] = os.environ['TEST_INPUT_INDEX']
     print(ii)

.github/workflows/run-individual-script-tests.yml

@@ -34,4 +34,4 @@ jobs:
         done
         pip install mlcflow
         mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
-        DOCKER_CM_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_CM_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 .github/scripts/process_individual_tests.py ${{ steps.getfile.outputs.files }}
+        DOCKER_MLC_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_MLC_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 .github/scripts/process_individual_tests.py ${{ steps.getfile.outputs.files }}

.github/workflows/test-amd-mlperf-inference-implementations.yml

@@ -19,7 +19,7 @@ jobs:
           if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
           python3 -m venv gh_action
           source gh_action/bin/activate
-          export CM_REPOS=$HOME/GH_CM
+          export MLC_REPOS=$HOME/GH_MLC
           pip install --upgrade cm4mlops
           cm pull repo
           cm run script --tags=run-mlperf,inference,_all-scenarios,_full,_r4.1-dev  --execution_mode=valid  --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c  --implementation=amd    --backend=pytorch    --category=datacenter --division=open --scenario=Offline  --docker_dt=yes --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --docker_cm_repo_branch=dev --adr.compiler.tags=gcc --device=rocm --use_dataset_from_host=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet --docker_skip_run_cmd=yes

.github/workflows/test-intel-mlperf-inference-implementations.yml

@@ -19,7 +19,7 @@ jobs:
           if [ -f "gh_action_conda/bin/deactivate" ]; then source gh_action_conda/bin/deactivate; fi
           python3 -m venv gh_action_conda
           source gh_action_conda/bin/activate
-          export CM_REPOS=$HOME/GH_CM
+          export MLC_REPOS=$HOME/GH_MLC
           pip install --upgrade cm4mlops
           pip install tabulate
           cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --execution_mode=valid  --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=IntelSPR.24c  --implementation=intel    --backend=pytorch    --category=datacenter --division=open --scenario=Offline  --docker_dt=yes --docker_it=no --docker_cm_repo=mlcommons@mlperf-automations --docker_cm_repo_branch=dev --adr.compiler.tags=gcc --device=cpu --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  --docker --quiet

.github/workflows/test-mlc-based-submission-generation.yml

@@ -67,21 +67,21 @@ jobs:
           extra_run_args=" --category=datacenter"
           description="Submission generation (system_meta.json not found in results folder)"
         elif [ "${{ matrix.case }}" == "closed" ]; then
-          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          extra_run_args=" --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
           description="Test submission - contains closed edge and datacenter"
         elif [ "${{ matrix.case }}" == "closed-no-compliance" ]; then
-          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          extra_run_args=" --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
           description="Test submission - contains closed edge and datacenter with no compliance tests"
         elif [ "${{ matrix.case }}" == "closed-power" ]; then
-          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          extra_run_args=" --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
           description="Test submission - contains closed-power edge and datacenter results"
         elif [ "${{ matrix.case }}" == "closed-failed-power-logs" ]; then
-          extra_run_args=" --env.CM_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
+          extra_run_args=" --env.MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS="--skip-extra-accuracy-files-check""
           description="Test submission - contains closed-power edge and datacenter results with failed power logs"
         fi
         # Dynamically set the log group to simulate a dynamic step name
         echo "::group::$description"
-        mlc ${{ matrix.action }} script --tags=generate,inference,submission  --version=v4.1 --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args
+        mlc ${{ matrix.action }} script --tags=generate,inference,submission  --version=v4.1 --clean --preprocess_submission=yes --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --env.MLC_TAR_OUTFILE=submission.tar.gz --division=${{ matrix.division }} --env.MLC_DETERMINE_MEMORY_CONFIGURATION=yes --quiet $extra_run_args
         exit_status=$?
         echo "Exit status for the job ${description} ${exit_status}"
         if [[ "${{ matrix.case }}" == "case-5" || "${{ matrix.case }}" == "case-6" ]]; then

.github/workflows/test-mlc-script-features.yml

@@ -63,10 +63,10 @@ jobs:
     - name: Run docker container from dockerhub on linux
       if: runner.os == 'linux'
       run: |
-        mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_cm_repo=mlcommons@mlperf-automations --image_name=cm-script-app-image-classification-onnx-py --env.CM_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.CM_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.CM_DOCKER_IMAGE_REPO=cknowledge --quiet
+        mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_cm_repo=mlcommons@mlperf-automations --image_name=cm-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=cknowledge --quiet
 
     - name: Run docker container locally on linux
       if: runner.os == 'linux'
       run: |
-        mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_cm_repo=mlcommons@mlperf-automations --image_name=mlc-script-app-image-classification-onnx-py --env.CM_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.CM_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.CM_DOCKER_IMAGE_REPO=local --quiet
+        mlc run script --tags=run,docker,container --adr.compiler.tags=gcc --docker_cm_repo=mlcommons@mlperf-automations --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet
 

.github/workflows/test-mlperf-inference-dlrm.yml

@@ -22,7 +22,7 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         python3 -m pip install cm4mlops
         cm pull repo
         cm run script --tags=run-mlperf,inference,_performance-only --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no  --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --clean
@@ -42,7 +42,7 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         python3 -m pip install cm4mlops
         cm pull repo
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --model=dlrm-v2-99 --implementation=intel --batch_size=1 --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }}  --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean

.github/workflows/test-mlperf-inference-gptj.yml

@@ -23,7 +23,7 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         python3 -m pip install cm4mlops
         cm pull repo
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --pull_changes=yes --pull_inference_changes=yes --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --docker_cm_repo_branch=dev --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --get_platform_details=yes --implementation=reference --clean

.github/workflows/test-mlperf-inference-llama2.yml

@@ -24,12 +24,12 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         pip install cm4mlops
         pip install tabulate 
         cm pull repo
         pip install "huggingface_hub[cli]"
         git config --global credential.helper store
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --precision=${{  matrix.precision }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }}  --docker --quiet --test_query_count=1 --target_qps=0.001 --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes  --clean
+        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --precision=${{  matrix.precision }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }}  --docker --quiet --test_query_count=1 --target_qps=0.001 --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.MLC_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes  --clean
         cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-mixtral.yml

@@ -25,11 +25,11 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         pip install cm4mlops
         pip install "huggingface_hub[cli]"
         git config --global credential.helper store
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
         cm pull repo
-        cm run script --tags=run-mlperf,inference,_submission,_short --adr.inference-src.tags=_branch.dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=3 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --adr.openorca-mbxp-gsm8k-combined-preprocessed.tags=_size.1
+        cm run script --tags=run-mlperf,inference,_submission,_short --adr.inference-src.tags=_branch.dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=3 --target_qps=0.001 --clean --env.MLC_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.MLC_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --adr.openorca-mbxp-gsm8k-combined-preprocessed.tags=_size.1
         cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -13,7 +13,7 @@ jobs:
   mlperf-inference-r50:
     runs-on: ${{ matrix.os }}
     env:
-      CM_INDEX: "on"
+      MLC_INDEX: "on"
     strategy:
       fail-fast: false
       matrix:
@@ -74,4 +74,4 @@ jobs:
         git commit -a -m "Test commit"
         git push https://x-access-token:${{ env.PAT }}@github.com/mlcommons/mlperf_inference_test_submissions_v5.0
 
-        # mlcr --tags=push,github,mlperf,inference,submission --env.CM_GITHUB_PAT=pat --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+        # mlcr --tags=push,github,mlperf,inference,submission --env.MLC_GITHUB_PAT=pat --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-mlperf-inference-rnnt.yml

@@ -30,7 +30,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies on Unix Platforms
       run: |
-        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+        MLC_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
     - name: Pull MLOps repository
       run: |
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}

.github/workflows/test-mlperf-inference-sdxl.yaml

@@ -18,8 +18,8 @@ jobs:
       run: |
         source gh_action/bin/deactivate || python3 -m venv gh_action
         source gh_action/bin/activate
-        export CM_REPOS=$HOME/GH_CM
+        export MLC_REPOS=$HOME/GH_MLC
         python3 -m pip install cm4mlops
         cm pull repo
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }}  --quiet --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions  --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
+        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }}  --quiet --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions  --env.MLC_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
         cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-tvm.yml

@@ -27,7 +27,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies on Unix Platforms
       run: |
-        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+        MLC_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
     - name: Pull MLOps repository
       run: |
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}

.github/workflows/test-nvidia-mlperf-inference-implementations.yml

@@ -47,8 +47,8 @@ jobs:
           if [ -f "gh_action/bin/deactivate" ]; then source gh_action/bin/deactivate; fi
           python3 -m venv gh_action
           source gh_action/bin/activate
-          export CM_REPOS=$HOME/GH_CM
-          CM_PULL_DEFAULT_MLOPS_REPO=no pip install --upgrade cm4mlops
+          export MLC_REPOS=$HOME/GH_MLC
+          MLC_PULL_DEFAULT_MLOPS_REPO=no pip install --upgrade cm4mlops
           cm pull repo
 
           cm run script --tags=run-mlperf,inference,_all-scenarios,_submission,_full,_r4.1-dev --preprocess_submission=yes --pull_changes=yes --pull_inference_changes=yes --execution_mode=valid --gpu_name=$gpu_name --pull_changes=yes --pull_inference_changes=yes --model=${{ matrix.model }} --submitter="MLCommons" --hw_name=$hw_name --implementation=nvidia --backend=tensorrt --category=datacenter,edge --division=closed  --docker_dt=yes --docker_it=no --docker_cm_repo=mlcommons@mlperf-automations --docker_cm_repo_branch=dev --adr.compiler.tags=gcc --device=cuda --use_model_from_host=yes --use_dataset_from_host=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean  $docker_string --quiet

.github/workflows/test-qaic-compute-sdk-build.yml

@@ -26,7 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+        MLC_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
         cm run script --tags=get,sys-utils-cm --quiet
 
     - name: Test QAIC Compute SDK for compilation