Merge pull request #105 from mlcommons/dev

Merge from dev

Author: anandhu-eng

.github/workflows/format.yml

@@ -53,10 +53,10 @@ jobs:
         run: |
           HAS_CHANGES=$(git diff --staged --name-only)
           if [ ${#HAS_CHANGES} -gt 0 ]; then
-            git config --global user.name mlcommons-bot
-            git config --global user.email "mlcommons-bot@users.noreply.github.com"
+            # Use the GitHub actor's name and email
+            git config --global user.name "${GITHUB_ACTOR}"
+            git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
             # Commit changes
             git commit -m '[Automated Commit] Format Codebase'
-            # Use the PAT to push changes
             git push 
           fi

.github/workflows/test-mlperf-inference-dlrm.yml

@@ -25,7 +25,7 @@ jobs:
         export CM_REPOS=$HOME/GH_CM
         python3 -m pip install cm4mlops
         cm pull repo
-        cm run script --tags=run-mlperf,inference,_performance-only --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --clean
+        cm run script --tags=run-mlperf,inference,_performance-only --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --model=dlrm-v2-99 --implementation=reference --backend=pytorch --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker --quiet --test_query_count=1 --target_qps=1 --docker_it=no  --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --clean
 
   build_intel:
     if: github.repository_owner == 'gateoverflow_off'

.github/workflows/test-mlperf-inference-gptj.yml

@@ -27,5 +27,5 @@ jobs:
         python3 -m pip install cm4mlops
         cm pull repo
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --pull_changes=yes --pull_inference_changes=yes --model=gptj-99 --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --docker_cm_repo_branch=dev --adr.compiler.tags=gcc --beam_size=1 --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --get_platform_details=yes --implementation=reference --clean
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=dev --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
 

.github/workflows/test-mlperf-inference-llama2.yml

@@ -32,4 +32,4 @@ jobs:
         git config --global credential.helper store
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=llama2-70b-99 --implementation=reference --backend=${{ matrix.backend }} --precision=${{  matrix.precision }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }}  --docker --quiet --test_query_count=1 --target_qps=0.001 --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --env.CM_MLPERF_MODEL_LLAMA2_70B_DOWNLOAD_TO_HOST=yes  --clean
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=dev --commit_message="Results from self hosted Github actions" --quiet --submission_dir=$HOME/gh_action_submissions
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-mixtral.yml

@@ -32,4 +32,4 @@ jobs:
         huggingface-cli login --token ${{ secrets.HF_TOKEN }} --add-to-git-credential
         cm pull repo
         cm run script --tags=run-mlperf,inference,_submission,_short --adr.inference-src.tags=_branch.dev --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --model=mixtral-8x7b --implementation=reference --batch_size=1 --precision=${{ matrix.precision }} --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --docker_it=no --docker_cm_repo=gateoverflow@mlperf-automations --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --docker --quiet --test_query_count=3 --target_qps=0.001 --clean --env.CM_MLPERF_MODEL_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --env.CM_MLPERF_DATASET_MIXTRAL_8X7B_DOWNLOAD_TO_HOST=yes --adr.openorca-mbxp-gsm8k-combined-preprocessed.tags=_size.1
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=dev --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from self hosted Github actions - GO-phoenix" --quiet --submission_dir=$HOME/gh_action_submissions

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -1,11 +1,10 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# Run MLPerf inference ResNet50
 
 name: MLPerf inference ResNet50
 
 on:
   pull_request_target:
-    branches: [ "main", "dev", "mlperf-inference" ]
+    branches: [ "main", "dev" ]
     paths:
       - '.github/workflows/test-mlperf-inference-resnet50.yml'
       - '**'
@@ -39,10 +38,20 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: |
         git config --system core.longpaths true
-    - name: Install dependencies
+
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+
+    - name: Pull MLOps repo
       run: |
-        pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+
     - name: Test MLPerf Inference ResNet50 (Windows)
       if: matrix.os == 'windows-latest'
       run: |
@@ -51,17 +60,19 @@ jobs:
       if: matrix.os != 'windows-latest'
       run: |
         cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
+    - name: Retrieve secrets from Keeper
+      id: ksecrets
+      uses: Keeper-Security/ksm-action@master
+      with:
+        keeper-secret-config: ${{ secrets.KSM_CONFIG }}
+        secrets: |-
+          ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT  # Fetch PAT and store in environment variable
+
     - name: Push Results
-      if: github.repository_owner == 'gateoverflow'
+      if: github.repository_owner == 'mlcommons'
       env:
-          USER: "GitHub Action"
-          EMAIL: "admin@gateoverflow.com"
-          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ env.PAT }}
       run: |
-        git config --global user.name "${{ env.USER }}"
-        git config --global user.email "${{ env.EMAIL }}"
-        git config --global credential.https://github.com.helper ""
-        git config --global credential.https://github.com.helper "!gh auth git-credential"
-        git config --global credential.https://gist.github.com.helper ""
-        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+        git config --global user.name mlcommons-bot
+        git config --global user.email "mlcommons-bot@users.noreply.github.com"
+        cm run script --tags=push,github,mlperf,inference,submission --env.CM_GITHUB_PAT=${{ env.PAT }} --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-mlperf-inference-retinanet.yml

@@ -1,11 +1,10 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# Run MLPerf inference Retinanet
 
 name: MLPerf inference retinanet
 
 on:
   pull_request_target:
-    branches: [ "main", "dev", "mlperf-inference" ]
+    branches: [ "main", "dev" ]
     paths:
       - '.github/workflows/test-mlperf-inference-retinanet.yml'
       - '**'
@@ -39,10 +38,18 @@ jobs:
       if: matrix.os == 'windows-latest'
       run: |
         git config --system core.longpaths true
-    - name: Install dependencies
+    - name: Install cm4mlops on Windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        $env:CM_PULL_DEFAULT_MLOPS_REPO = "no"; pip install cm4mlops
+    - name: Install dependencies on Unix Platforms
+      if: matrix.os != 'windows-latest'
+      run: |
+        CM_PULL_DEFAULT_MLOPS_REPO=no pip install cm4mlops
+    - name: Pull MLOps repo
       run: |
-        python3 -m pip install "cmind @ git+https://git@github.com/mlcommons/ck.git@mlperf-inference#subdirectory=cm"
         cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+
     - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }}
       if: matrix.os == 'windows-latest'
       run: |

HISTORY.md

@@ -0,0 +1,72 @@
+## Timeline of CM developments
+
+### **🚀 2022: Foundation and Early Developments**
+
+- **March 2022:** Grigori Fursin began developing **CM (Collective Mind)**, also referred to as **CK2**, as a successor to CK [at OctoML](https://github.com/octoml/ck/commits/master/?since=2022-03-01&until=2022-03-31).
+- **April 2022:** **Arjun Suresh** joined OctoML and collaborated with Grigori on developing **CM Automation** tools.
+- **May 2022:** The **CM CLI** and **Python interface** were successfully [implemented and stabilized](https://github.com/octoml/ck/commits/master/?since=2022-04-01&until=2022-05-31) by Grigori.
+
+---
+
+### **🛠️ July–September 2022: MLPerf Integration and First Submission**
+
+- Arjun completed the development of the **MLPerf Inference Script** within CM.
+- OctoML achieved **first MLPerf Inference submission (v2.1)** using **CM Automation** ([progress here](https://github.com/octoml/ck/commits/master/?since=2022-06-01&until=2022-09-30)).
+
+---
+
+### **📊 October 2022 – March 2023: End-to-End Automation**
+
+- End-to-end MLPerf inference automations using CM was successfully [completed in CM](https://github.com/octoml/ck/commits/master/?since=2022-10-01&until=2023-03-31).
+- **Additional benchmarks** and **Power Measurement support** were integrated into CM.
+- **cTuning** achieved a successful MLPerf Inference **v3.0 submission** using CM Automation.
+
+---
+
+### **🔄 April 2023: Transition and New Funding**
+
+- Arjun and Grigori departed OctoML and resumed **CM development** under funding from **cKnowledge.org** and **cTuning**.
+
+---
+
+### **🚀 April–October 2023: Expanded Support and Milestone Submission**
+
+- MLPerf inference automations were [extended](https://github.com/mlcommons/ck/commits/master?since=2023-04-01&until=2023-10-31) to support **NVIDIA implementations**.
+- **cTuning** achieved the **largest-ever MLPerf Inference submission (v3.1)** using CM Automation.
+
+---
+
+### **🤝 November 2023: MLCommons Partnership**
+
+- **MLCommons** began funding CM development to enhance support for **NVIDIA MLPerf inference** and introduce support for **Intel** and **Qualcomm MLPerf inference** implementations.
+
+---
+
+### **🌐 October 2023 – March 2024: Multi-Platform Expansion**
+
+- MLPerf inference automations were [expanded](https://github.com/mlcommons/ck/commits/master?since=2023-10-01&until=2024-03-15) to support **NVIDIA, Intel, and Qualcomm implementations**.
+- **cTuning** completed the **MLPerf Inference v4.0 submission** using CM Automation.
+
+---
+
+### **📝 April 2024: Documentation Improvements**
+
+- MLCommons contracted **Arjun Suresh** via **GATEOverflow** to improve **MLPerf inference documentation** and enhance CM Automation on various platforms.
+
+---
+
+### **👥 May 2024: Team Expansion**
+
+- **Anandhu Sooraj** joined MLCommons to collaborate with **Arjun Suresh** on CM development.
+
+---
+
+### **📖 June–December 2024: Enhanced Documentation and Automation**
+
+- **Dedicated documentation site** launched for **MLPerf inference**.
+- **CM scripts** were developed for **MLPerf Automotive**.
+- **CM Docker support** was stabilized.
+- **GitHub Actions workflows** were added for **MLPerf inference reference implementations** and **NVIDIA integrations** ([see updates](https://github.com/mlcommons/mlperf-automations/commits/main?since=2024-06-01&until=2024-12-31)).
+
+---
+

VERSION

@@ -1 +1 @@
-0.6.18
+0.6.19

git_commit_hash.txt

@@ -1 +1 @@
-76796b4c3966b04011c3cb6118412516c90ba50b
+81816f94c4a396a012412cb3a1cf4096b4ad103e