Update customize.py | Use Path instead of string in inference-submiss…

.github/workflows/build_wheel.yml

@@ -11,6 +11,7 @@ on:
       - VERSION
 
 jobs:
+
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/format.yml

@@ -59,6 +59,6 @@ jobs:
             git config --global user.name github-actions[bot]
             git config --global user.email "github-actions[bot]@users.noreply.github.com"
             # Commit changes
-            git commit -m '[Automated Commit] Format Codebase'
+            git commit -m '[Automated Commit] Format Codebase [skip ci]'
             git push 
           fi

script/activate-python-venv/customize.py

@@ -1,4 +1,5 @@
 from mlc import utils
+
 import os
 
 

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -250,9 +250,9 @@ def get_run_cmd_reference(
                 'MLC_MLPERF_DEVICE') != "tpu":
             if os_info['platform'] == 'windows':
                 cmd = "python python/main.py --profile " + env['MLC_MODEL'] + "-" + env['MLC_MLPERF_BACKEND'] + \
-                    " --model=" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + ' --dataset-path=' + env['MLC_DATASET_PREPROCESSED_PATH'] + \
-                    " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \
-                    " --output " + env['OUTPUT_DIR'] + " " + \
+                    " --model=\"" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + '" --dataset-path="' + env['MLC_DATASET_PREPROCESSED_PATH'] + \
+                    "\" --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \
+                    " --output \"" + env['OUTPUT_DIR'] + "\" " + \
                     env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
                     scenario_extra_options + mode_extra_options + dataset_options
             else:

script/draw-graph-from-json-data/customize.py

@@ -13,14 +13,15 @@ def preprocess(i):
     automation = i['automation']
 
     quiet = (env.get('MLC_QUIET', False) == 'yes')
+    q = '"' if os_info['platform'] == 'windows' else "'"
 
-    env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'],"process-mlc-deps.py")}  {env['MLC_JSON_INPUT_FILE']}"""
+    env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'],"process-mlc-deps.py")}{q} {q}{env['MLC_JSON_INPUT_FILE']}{q} """
 
     if env.get('MLC_OUTPUT_IMAGE_PATH', '') != '':
-        env['MLC_RUN_CMD'] += f""" --output_image {env['MLC_OUTPUT_IMAGE_PATH']}"""
+        env['MLC_RUN_CMD'] += f""" --output_image {q}{env['MLC_OUTPUT_IMAGE_PATH']}{q} """
 
     if env.get('MLC_OUTPUT_MERMAID_PATH', '') != '':
-        env['MLC_RUN_CMD'] += f""" --output_mermaid {env['MLC_OUTPUT_MERMAID_PATH']}"""
+        env['MLC_RUN_CMD'] += f""" --output_mermaid {q}{env['MLC_OUTPUT_MERMAID_PATH']}{q} """
 
     return {'return': 0}
 

script/generate-mlperf-inference-submission/customize.py

@@ -5,6 +5,7 @@
 import sys
 from tabulate import tabulate
 import mlperf_utils
+from pathlib import Path
 
 
 def preprocess(i):
@@ -71,7 +72,6 @@ def generate_submission(env, state, inp, submission_division):
     sys.path.append(submission_checker_dir)
 
     if env.get('MLC_MLPERF_INFERENCE_SUBMISSION_DIR', '') == '':
-        from pathlib import Path
         user_home = str(Path.home())
         env['MLC_MLPERF_INFERENCE_SUBMISSION_DIR'] = os.path.join(
             user_home, "mlperf_submission")
@@ -275,15 +275,19 @@ def generate_submission(env, state, inp, submission_division):
                             {model: returned_model_name})
 
         if check_dict_filled(sut_info.keys(), sut_info):
-            system = env.get('MLC_HW_NAME', sut_info["system_name"])
+            system = env.get(
+                'MLC_HW_NAME',
+                sut_info["system_name"]).replace(
+                " ",
+                "_")
             implementation = sut_info["implementation"]
             device = sut_info["device"]
             framework = sut_info["framework"].replace(" ", "_")
             framework_version = sut_info["framework_version"]
             run_config = sut_info["run_config"]
             new_res = f"{system}-{implementation}-{device}-{framework}-{run_config}"
         else:
-            new_res = res
+            new_res = res.replace(" ", "_")
 
         print(f"The SUT folder name for submission generation is: {new_res}")
 
@@ -535,17 +539,19 @@ def generate_submission(env, state, inp, submission_division):
                             measurements_json = json.load(f)
                             model_precision = measurements_json.get(
                                 "weight_data_types", "fp32")
-                        shutil.copy(
-                            measurements_json_path,
-                            os.path.join(
-                                target_measurement_json_path,
-                                sub_res + '.json'))
 
-                        shutil.copy(
-                            measurements_json_path,
-                            os.path.join(
-                                target_measurement_json_path,
-                                'model-info.json'))
+                        # Convert paths to Path objects
+                        measurements_json_path = Path(measurements_json_path)
+                        target_measurement_json_path = Path(
+                            target_measurement_json_path)
+
+                        destination = Path(
+                            target_measurement_json_path) / f"{sub_res}.json"
+                        shutil.copy(measurements_json_path, destination)
+                        destination = Path(
+                            target_measurement_json_path) / "model-info.json"
+                        shutil.copy(measurements_json_path, destination)
+
                     else:
                         if mode.lower() == "performance":
                             return {

script/get-mlperf-inference-sut-configs/customize.py

@@ -13,7 +13,7 @@ def postprocess(i):
             'MLC_HOST_SYSTEM_NAME',
             'default').replace(
             "-",
-            "_")
+            "_").replace(" ", "_")
         env['MLC_HW_NAME'] = host_name
 
     device = env.get('MLC_MLPERF_DEVICE', 'cpu')

script/get-mlperf-inference-sut-description/customize.py

@@ -17,7 +17,7 @@ def preprocess(i):
             'MLC_HOST_SYSTEM_NAME',
             'default').replace(
             "-",
-            "_")
+            "_").replace(" ", "_")
         env['MLC_HW_NAME'] = host_name
         auto_detected_hw_name = True
 

script/get-platform-details/run.sh

@@ -2,137 +2,137 @@
 
 OUTPUT_FILE="$MLC_PLATFORM_DETAILS_FILE_PATH"
 #set -e
-#echo $OUTPUT_FILE
+#echo "${OUTPUT_FILE}"
 echo "WARNING: sudo permission is needed for some of the below commands"
 
-echo "Platform Details" > $OUTPUT_FILE
-echo "" >> $OUTPUT_FILE
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
-echo "1. uname -a" >> $OUTPUT_FILE
-eval "uname -a" >> $OUTPUT_FILE
+echo "Platform Details" > "${OUTPUT_FILE}"
+echo "" >> "${OUTPUT_FILE}"
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
+echo "1. uname -a" >> "${OUTPUT_FILE}"
+eval "uname -a" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "2. w" >> $OUTPUT_FILE
-eval "w" >> $OUTPUT_FILE
+echo "2. w" >> "${OUTPUT_FILE}"
+eval "w" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "3. Username" >> $OUTPUT_FILE
-eval "whoami" >> $OUTPUT_FILE
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "3. Username" >> "${OUTPUT_FILE}"
+eval "whoami" >> "${OUTPUT_FILE}"
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "4. ulimit -a" >> $OUTPUT_FILE
-eval "ulimit -a" >> $OUTPUT_FILE
+echo "4. ulimit -a" >> "${OUTPUT_FILE}"
+eval "ulimit -a" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "5. sysinfo process ancestry" >> $OUTPUT_FILE
-eval "pstree" >> $OUTPUT_FILE
+echo "5. sysinfo process ancestry" >> "${OUTPUT_FILE}"
+eval "pstree" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "6. /proc/cpuinfo" >> $OUTPUT_FILE
-eval "cat /proc/cpuinfo" >> $OUTPUT_FILE
+echo "6. /proc/cpuinfo" >> "${OUTPUT_FILE}"
+eval "cat /proc/cpuinfo" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "7. lscpu" >> $OUTPUT_FILE
-eval "lscpu" >> $OUTPUT_FILE
+echo "7. lscpu" >> "${OUTPUT_FILE}"
+eval "lscpu" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "8. numactl --hardware" >> $OUTPUT_FILE
+echo "8. numactl --hardware" >> "${OUTPUT_FILE}"
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
     echo "${MLC_SUDO} numactl --hardware"
-    eval "${MLC_SUDO} numactl --hardware" >> $OUTPUT_FILE
+    eval "${MLC_SUDO} numactl --hardware" >> "${OUTPUT_FILE}"
     test $? -eq 0 || exit $?
 else
-    echo "Requires SUDO permission" >> $OUTPUT_FILE
+    echo "Requires SUDO permission" >> "${OUTPUT_FILE}"
 fi
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "9. /proc/meminfo" >> $OUTPUT_FILE
-eval "cat /proc/meminfo" >> $OUTPUT_FILE
+echo "9. /proc/meminfo" >> "${OUTPUT_FILE}"
+eval "cat /proc/meminfo" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "10. who -r" >> $OUTPUT_FILE
-eval "who -r" >> $OUTPUT_FILE
+echo "10. who -r" >> "${OUTPUT_FILE}"
+eval "who -r" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "11. Systemd service manager version" >> $OUTPUT_FILE
-eval "systemctl --version | head -n 1" >> $OUTPUT_FILE
+echo "11. Systemd service manager version" >> "${OUTPUT_FILE}"
+eval "systemctl --version | head -n 1" >> "${OUTPUT_FILE}"
 #test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE
-eval "systemctl list-unit-files" >> $OUTPUT_FILE
+echo "12. Services, from systemctl list-unit-files" >> "${OUTPUT_FILE}"
+eval "systemctl list-unit-files" >> "${OUTPUT_FILE}"
 #test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE
-eval "cat /proc/cmdline" >> $OUTPUT_FILE
+echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> "${OUTPUT_FILE}"
+eval "cat /proc/cmdline" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "14. cpupower frequency-info" >> $OUTPUT_FILE
-eval "cpupower frequency-info" >> $OUTPUT_FILE
-test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> $OUTPUT_FILE
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "14. cpupower frequency-info" >> "${OUTPUT_FILE}"
+eval "cpupower frequency-info" >> "${OUTPUT_FILE}"
+test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> "${OUTPUT_FILE}"
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "15. sysctl" >> $OUTPUT_FILE
+echo "15. sysctl" >> "${OUTPUT_FILE}"
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
     echo "${MLC_SUDO} sysctl -a"
-    eval "${MLC_SUDO} sysctl -a" >> $OUTPUT_FILE
+    eval "${MLC_SUDO} sysctl -a" >> "${OUTPUT_FILE}"
     test $? -eq 0 || exit $?
 else
-    echo "Requires SUDO permission" >> $OUTPUT_FILE
+    echo "Requires SUDO permission" >> "${OUTPUT_FILE}"
 fi
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE
-eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> $OUTPUT_FILE
+echo "16. /sys/kernel/mm/transparent_hugepage" >> "${OUTPUT_FILE}"
+eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> $OUTPUT_FILE
-eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> $OUTPUT_FILE
+echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> "${OUTPUT_FILE}"
+eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "18. OS release" >> $OUTPUT_FILE
-eval "cat /etc/os-release" >> $OUTPUT_FILE
+echo "18. OS release" >> "${OUTPUT_FILE}"
+eval "cat /etc/os-release" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "19. Disk information" >> $OUTPUT_FILE
-eval "lsblk" >> $OUTPUT_FILE
+echo "19. Disk information" >> "${OUTPUT_FILE}"
+eval "lsblk" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "20. /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE
-eval "ls /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE
+echo "20. /sys/devices/virtual/dmi/id" >> "${OUTPUT_FILE}"
+eval "ls /sys/devices/virtual/dmi/id" >> "${OUTPUT_FILE}"
 test $? -eq 0 || exit $?
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "21. dmidecode" >> $OUTPUT_FILE
+echo "21. dmidecode" >> "${OUTPUT_FILE}"
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
-    eval "${MLC_SUDO} dmidecode" >> $OUTPUT_FILE
-    test $? -eq 0 || echo "FAILED: dmidecode" >> $OUTPUT_FILE
+    eval "${MLC_SUDO} dmidecode" >> "${OUTPUT_FILE}"
+    test $? -eq 0 || echo "FAILED: dmidecode" >> "${OUTPUT_FILE}"
 else
-    echo "Requires SUDO permission" >> $OUTPUT_FILE
+    echo "Requires SUDO permission" >> "${OUTPUT_FILE}"
 fi
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "22. BIOS" >> $OUTPUT_FILE
+echo "22. BIOS" >> "${OUTPUT_FILE}"
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
-    eval "${MLC_SUDO} dmidecode -t bios" >> $OUTPUT_FILE
-    test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> $OUTPUT_FILE
+    eval "${MLC_SUDO} dmidecode -t bios" >> "${OUTPUT_FILE}"
+    test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> "${OUTPUT_FILE}"
 else
-    echo "Requires SUDO permission" >> $OUTPUT_FILE
+    echo "Requires SUDO permission" >> "${OUTPUT_FILE}"
 fi
-echo "------------------------------------------------------------" >> $OUTPUT_FILE
+echo "------------------------------------------------------------" >> "${OUTPUT_FILE}"
 
-echo "System information has been saved to $OUTPUT_FILE"
+echo "System information has been saved to "${OUTPUT_FILE}""