Sync <- Dev

.github/workflows/test-cm-based-submission-generation.yml

@@ -62,6 +62,7 @@ jobs:
     - name: Pull repo where test cases are uploaded
       run: |
         git clone -b submission-generation-tests https://github.com/mlcommons/inference.git submission_generation_tests
+
     - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} 
       continue-on-error: true
       run: |

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -67,12 +67,11 @@ jobs:
         keeper-secret-config: ${{ secrets.KSM_CONFIG }}
         secrets: |-
           ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT  # Fetch PAT and store in environment variable
-
     - name: Push Results
-      if: github.repository_owner == 'mlcommons_off'
       env:
-          GITHUB_TOKEN: ${{ env.PAT }}
+        PAT: ${{ env.PAT }}
+      if: github.repository_owner == 'mlcommons'
       run: |
         git config --global user.name mlcommons-bot
         git config --global user.email "mlcommons-bot@users.noreply.github.com"
-        cm run script --tags=push,github,mlperf,inference,submission --env.CM_GITHUB_PAT=${{ env.PAT }} --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+        cm run script --tags=push,github,mlperf,inference,submission --env.CM_GITHUB_PAT=$PAT --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-nvidia-mlperf-inference-implementations.yml

@@ -17,8 +17,8 @@ jobs:
       strategy:
         fail-fast: false
         matrix:
-          # system: [ "GO-spr", "phoenix-Amd-Am5", "GO-i9", "mlc-server" ]
-          system: [ "mlc-server" ] 
+          system: [ "GO-spr", "phoenix-Amd-Am5", "GO-i9"]
+          # system: [ "mlc-server" ] 
           python-version: [ "3.12" ]
           model: [ "resnet50",  "retinanet",  "bert-99", "bert-99.9", "gptj-99.9", "3d-unet-99.9", "sdxl" ]
           exclude:
@@ -33,7 +33,7 @@ jobs:
           if [ "${{ matrix.system }}" = "GO-spr" ]; then
             hw_name="RTX4090x2"
             gpu_name=rtx_4090
-            docker_string=" --docker"
+            docker_string=" --docker --docker_recreate=yes"
           elif [ "${{ matrix.system }}" = "mlc-server" ]; then
             hw_name="H100x8"
             gpu_name=h100

CONTRIBUTORS.md

@@ -8,7 +8,7 @@ Thank you for your interest in contributing to **MLPerf Automations**! We welcom
 
 We value all contributions, whether they are code, documentation, bug reports, or feature suggestions. If you contribute **more than 50 lines of code** (including tests and documentation), you will be officially recognized as a project contributor.  
 
-**Note:** Trivial contributions, such as minor typo fixes or small formatting changes, will not count toward the 50-line threshold.  
+**Note:** Trivial contributions, such as minor typo fixes or small formatting changes, and documentation changes will not count toward the 50-line threshold.  
 
 To contribute:  
 1. **Fork** the repository.  
@@ -29,7 +29,7 @@ Once your contribution exceeds 50 lines of code (in total), we will:
 
 ## Current Contributors
 
-- **Grigori Fursin** - *Initial Development, CLI workflow support via CMind, Added core automation features*
+- **Grigori Fursin** - *Initial Development, CMind development to drive the automations, Added core automation features*
 - **Arjun Suresh** - *Initial Development, Added core automation features* 
 - **Anandhu Sooraj** - *Added multiple CM scripts for MLPerf Inference*
 - **Thomaz Zhu** - *Added CPP implementation for MLPerf Inference Onnxruntime*

VERSION

@@ -1 +1 @@
-0.6.19
+0.6.21

automation/script/module.py

@@ -2183,27 +2183,6 @@ def _run(self, i):
         if print_readme or repro_prefix != '':
             readme = self._get_readme(cmd, run_state)
 
-        # Copy Docker sample
-        if repro_prefix != '' and repro_dir != '':
-            docker_template_path = os.path.join(
-                self.path, 'docker_repro_example')
-            if os.path.isdir(docker_template_path):
-                try:
-                    shutil.copytree(
-                        docker_template_path,
-                        repro_dir,
-                        dirs_exist_ok=True)
-                except Exception as e:
-                    pass
-
-            docker_container = self._get_docker_container(cmd, run_state)
-
-            try:
-                with open(os.path.join(repro_dir, 'ubuntu-23.04.Dockerfile'), 'a+') as f:
-                    f.write(docker_container)
-            except BaseException:
-                pass
-
         if print_readme:
             with open('README-cm.md', 'w') as f:
                 f.write(readme)
@@ -3808,26 +3787,23 @@ def _get_readme(self, cmd_parts, run_state):
         content = ''
 
         content += """
-*This README was automatically generated by the [CM framework](https://github.com/mlcommons/ck).*
+*This README was automatically generated.*
 
 ## Install CM
 
 ```bash
-pip install cmind -U
+pip install cm4mlops
 ```
 
 Check [this readme](https://github.com/mlcommons/ck/blob/master/docs/installation.md)
 with more details about installing CM and dependencies across different platforms
 (Ubuntu, MacOS, Windows, RHEL, ...).
 
-## Install CM automation repositories
-
-```bash
-cm pull repo mlcommons@cm4mlops --checkout=dev
 """
 
         current_cm_repo = run_state['script_repo_alias']
-        if current_cm_repo not in ['mlcommons@ck', 'mlcommons@cm4mlops']:
+        if current_cm_repo not in [
+                'mlcommons@mlperf-automations', 'mlcommons@cm4mlops']:
             content += '\ncm pull repo ' + \
                 run_state['script_repo_alias'] + '\n'
 
@@ -3867,57 +3843,6 @@ def _get_readme(self, cmd_parts, run_state):
 
         return content
 
-    ##########################################################################
-    def _get_docker_container(self, cmd_parts, run_state):
-        """
-        Outputs a Markdown README file listing the CM run commands for the dependencies
-        """
-
-        deps = run_state['deps']
-
-        version_info = run_state.get('version_info', [])
-        version_info_dict = {}
-
-        for v in version_info:
-            k = list(v.keys())[0]
-            version_info_dict[k] = v[k]
-
-        content = ''
-
-        content += """
-
-# The following CM commands were automatically generated (prototype)
-
-cm pull repo mlcommons@cm4mlops --checkout=dev
-
-"""
-        current_cm_repo = run_state['script_repo_alias']
-        if current_cm_repo not in ['mlcommons@ck', 'mlcommons@cm4mlops']:
-            content += '\ncm pull repo ' + \
-                run_state['script_repo_alias'] + '\n\n'
-
-        deps_ = ''
-
-        for dep_tags in deps:
-
-            xversion = ''
-            version = version_info_dict.get(dep_tags, {}).get('version', '')
-            if version != '':
-                xversion = ' --version={}\n'.format(version)
-
-            content += "# cm run script --tags=" + \
-                dep_tags + "{}\n\n".format(xversion)
-
-        cmd = "cm run script "
-
-        for cmd_part in cmd_parts:
-            x = '"' if ' ' in cmd_part and not cmd_part.startswith('-') else ''
-            cmd = cmd + " " + x + cmd_part + x
-
-        content += cmd + '\n'
-
-        return content
-
     ##########################################################################
 
     def _print_versions(self, run_state):
@@ -4771,41 +4696,6 @@ def doc(self, i):
         return utils.call_internal_module(
             self, __file__, 'module_misc', 'doc', i)
 
-    ############################################################
-    def gui(self, i):
-        """
-        Run GUI for CM script.
-
-        Args:
-          (CM input dict):
-
-        Returns:
-          (CM return dict):
-
-          * return (int): return code == 0 if no error and >0 if error
-          * (error) (str): error string if return>0
-
-        """
-
-        artifact = i.get('artifact', '')
-        tags = ''
-        if artifact != '':
-            if ' ' in artifact:
-                tags = artifact.replace(' ', ',')
-
-        if tags == '':
-            tags = i.get('tags', '')
-
-        if 'tags' in i:
-            del (i['tags'])
-
-        i['action'] = 'run'
-        i['artifact'] = 'gui'
-        i['parsed_artifact'] = [('gui', '605cac42514a4c69')]
-        i['script'] = tags.replace(',', ' ')
-
-        return self.cmind.access(i)
-
     ############################################################
 
     def dockerfile(self, i):
@@ -4873,7 +4763,7 @@ def docker(self, i):
           (docker_os_version) (str): force docker OS version (default: 22.04)
           (docker_image_tag_extra) (str): add extra tag (default:-latest)
 
-          (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops)
+          (docker_cm_repo) (str): force CM automation repository when building Docker (default: mlperf-automations)
           (docker_cm_repos)
           (docker_cm_repo_flags)
 
@@ -5644,20 +5534,13 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
                         script_repo_alias.replace('@', '/') + '/issues'
 
             if repo_to_report == '':
-                repo_to_report = 'https://github.com/mlcommons/cm4mlops/issues'
+                repo_to_report = 'https://github.com/mlcommons/mlperf-automations/issues'
 
             note = '''
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Note that it is often a portability issue of a third-party tool or a native script
-wrapped and unified by this CM script (automation recipe). Please re-run
-this script with --repro flag and report this issue with the original
-command line, cm-repro directory and full log here:
-
-{}
-
-The CM concept is to collaboratively fix such issues inside portable CM scripts
-to make existing tools and native scripts more portable, interoperable
-and deterministic. Thank you'''.format(repo_to_report)
+Please file an issue at {} along with the full CM command being run and the relevant
+or full console log.
+'''.format(repo_to_report)
 
             rr = {
                 'return': 2,

git_commit_hash.txt

@@ -1 +1 @@
-81816f94c4a396a012412cb3a1cf4096b4ad103e
+e686cec959f5cf999f8778dc1c3b8a9af04eb823

script/activate-python-venv/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/add-custom-nvidia-system/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2023-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-image-classification-onnx-py/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-image-classification-tf-onnx-cpp/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2023-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-image-classification-torch-py/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-image-classification-tvm-onnx-py/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-image-corner-detection/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-loadgen-generic-python/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2022-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-mlperf-automotive-mlcommons-python/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2024-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-mlperf-automotive/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2024-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-mlperf-inference-ctuning-cpp-tflite/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2024-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-mlperf-inference-dummy/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2024-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.

script/app-mlperf-inference-intel/COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright Notice
+
+© 2024-2025 MLCommons. All Rights Reserved.
+
+This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at:
+
+[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License.