diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 64f1789e4..ba82117cf 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -11,6 +11,7 @@ on: - VERSION jobs: + build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8523dbfc5..3fd479b11 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -59,6 +59,6 @@ jobs: git config --global user.name github-actions[bot] git config --global user.email "github-actions[bot]@users.noreply.github.com" # Commit changes - git commit -m '[Automated Commit] Format Codebase' + git commit -m '[Automated Commit] Format Codebase [skip ci]' git push fi diff --git a/script/activate-python-venv/customize.py b/script/activate-python-venv/customize.py index 2740bb9d8..20ee8f2cf 100644 --- a/script/activate-python-venv/customize.py +++ b/script/activate-python-venv/customize.py @@ -1,4 +1,5 @@ from mlc import utils + import os diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index b7c0a5efc..df85eb7e1 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -250,9 +250,9 @@ def get_run_cmd_reference( 'MLC_MLPERF_DEVICE') != "tpu": if os_info['platform'] == 'windows': cmd = "python python/main.py --profile " + env['MLC_MODEL'] + "-" + env['MLC_MLPERF_BACKEND'] + \ - " --model=" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + ' --dataset-path=' + env['MLC_DATASET_PREPROCESSED_PATH'] + \ - " --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \ - " --output " + env['OUTPUT_DIR'] + " " + \ + " --model=\"" + env['MLC_ML_MODEL_FILE_WITH_PATH'] + '" --dataset-path="' + env['MLC_DATASET_PREPROCESSED_PATH'] + \ + "\" --scenario " + env['MLC_MLPERF_LOADGEN_SCENARIO'] + " " + \ + " --output \"" + env['OUTPUT_DIR'] + "\" " + \ env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ scenario_extra_options + mode_extra_options + dataset_options else: diff --git a/script/draw-graph-from-json-data/customize.py b/script/draw-graph-from-json-data/customize.py index 77affa7e3..56c734da3 100644 --- a/script/draw-graph-from-json-data/customize.py +++ b/script/draw-graph-from-json-data/customize.py @@ -13,14 +13,15 @@ def preprocess(i): automation = i['automation'] quiet = (env.get('MLC_QUIET', False) == 'yes') + q = '"' if os_info['platform'] == 'windows' else "'" - env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'],"process-mlc-deps.py")} {env['MLC_JSON_INPUT_FILE']}""" + env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {q}{os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'],"process-mlc-deps.py")}{q} {q}{env['MLC_JSON_INPUT_FILE']}{q} """ if env.get('MLC_OUTPUT_IMAGE_PATH', '') != '': - env['MLC_RUN_CMD'] += f""" --output_image {env['MLC_OUTPUT_IMAGE_PATH']}""" + env['MLC_RUN_CMD'] += f""" --output_image {q}{env['MLC_OUTPUT_IMAGE_PATH']}{q} """ if env.get('MLC_OUTPUT_MERMAID_PATH', '') != '': - env['MLC_RUN_CMD'] += f""" --output_mermaid {env['MLC_OUTPUT_MERMAID_PATH']}""" + env['MLC_RUN_CMD'] += f""" --output_mermaid {q}{env['MLC_OUTPUT_MERMAID_PATH']}{q} """ return {'return': 0} diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 3245e1893..0d1ac5a69 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -5,6 +5,7 @@ import sys from tabulate import tabulate import mlperf_utils +from pathlib import Path def preprocess(i): @@ -71,7 +72,6 @@ def generate_submission(env, state, inp, submission_division): sys.path.append(submission_checker_dir) if env.get('MLC_MLPERF_INFERENCE_SUBMISSION_DIR', '') == '': - from pathlib import Path user_home = str(Path.home()) env['MLC_MLPERF_INFERENCE_SUBMISSION_DIR'] = os.path.join( user_home, "mlperf_submission") @@ -275,7 +275,11 @@ def generate_submission(env, state, inp, submission_division): {model: returned_model_name}) if check_dict_filled(sut_info.keys(), sut_info): - system = env.get('MLC_HW_NAME', sut_info["system_name"]) + system = env.get( + 'MLC_HW_NAME', + sut_info["system_name"]).replace( + " ", + "_") implementation = sut_info["implementation"] device = sut_info["device"] framework = sut_info["framework"].replace(" ", "_") @@ -283,7 +287,7 @@ def generate_submission(env, state, inp, submission_division): run_config = sut_info["run_config"] new_res = f"{system}-{implementation}-{device}-{framework}-{run_config}" else: - new_res = res + new_res = res.replace(" ", "_") print(f"The SUT folder name for submission generation is: {new_res}") @@ -535,17 +539,19 @@ def generate_submission(env, state, inp, submission_division): measurements_json = json.load(f) model_precision = measurements_json.get( "weight_data_types", "fp32") - shutil.copy( - measurements_json_path, - os.path.join( - target_measurement_json_path, - sub_res + '.json')) - shutil.copy( - measurements_json_path, - os.path.join( - target_measurement_json_path, - 'model-info.json')) + # Convert paths to Path objects + measurements_json_path = Path(measurements_json_path) + target_measurement_json_path = Path( + target_measurement_json_path) + + destination = Path( + target_measurement_json_path) / f"{sub_res}.json" + shutil.copy(measurements_json_path, destination) + destination = Path( + target_measurement_json_path) / "model-info.json" + shutil.copy(measurements_json_path, destination) + else: if mode.lower() == "performance": return { diff --git a/script/get-mlperf-inference-sut-configs/customize.py b/script/get-mlperf-inference-sut-configs/customize.py index e8b1e7be7..a2f605ae5 100644 --- a/script/get-mlperf-inference-sut-configs/customize.py +++ b/script/get-mlperf-inference-sut-configs/customize.py @@ -13,7 +13,7 @@ def postprocess(i): 'MLC_HOST_SYSTEM_NAME', 'default').replace( "-", - "_") + "_").replace(" ", "_") env['MLC_HW_NAME'] = host_name device = env.get('MLC_MLPERF_DEVICE', 'cpu') diff --git a/script/get-mlperf-inference-sut-description/customize.py b/script/get-mlperf-inference-sut-description/customize.py index aaadf5be0..93746764e 100644 --- a/script/get-mlperf-inference-sut-description/customize.py +++ b/script/get-mlperf-inference-sut-description/customize.py @@ -17,7 +17,7 @@ def preprocess(i): 'MLC_HOST_SYSTEM_NAME', 'default').replace( "-", - "_") + "_").replace(" ", "_") env['MLC_HW_NAME'] = host_name auto_detected_hw_name = True diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 2336146e1..843d1cba7 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -2,137 +2,137 @@ OUTPUT_FILE="$MLC_PLATFORM_DETAILS_FILE_PATH" #set -e -#echo $OUTPUT_FILE +#echo "${OUTPUT_FILE}" echo "WARNING: sudo permission is needed for some of the below commands" -echo "Platform Details" > $OUTPUT_FILE -echo "" >> $OUTPUT_FILE -echo "------------------------------------------------------------" >> $OUTPUT_FILE -echo "1. uname -a" >> $OUTPUT_FILE -eval "uname -a" >> $OUTPUT_FILE +echo "Platform Details" > "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" +echo "1. uname -a" >> "${OUTPUT_FILE}" +eval "uname -a" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "2. w" >> $OUTPUT_FILE -eval "w" >> $OUTPUT_FILE +echo "2. w" >> "${OUTPUT_FILE}" +eval "w" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "3. Username" >> $OUTPUT_FILE -eval "whoami" >> $OUTPUT_FILE -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "3. Username" >> "${OUTPUT_FILE}" +eval "whoami" >> "${OUTPUT_FILE}" +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "4. ulimit -a" >> $OUTPUT_FILE -eval "ulimit -a" >> $OUTPUT_FILE +echo "4. ulimit -a" >> "${OUTPUT_FILE}" +eval "ulimit -a" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "5. sysinfo process ancestry" >> $OUTPUT_FILE -eval "pstree" >> $OUTPUT_FILE +echo "5. sysinfo process ancestry" >> "${OUTPUT_FILE}" +eval "pstree" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "6. /proc/cpuinfo" >> $OUTPUT_FILE -eval "cat /proc/cpuinfo" >> $OUTPUT_FILE +echo "6. /proc/cpuinfo" >> "${OUTPUT_FILE}" +eval "cat /proc/cpuinfo" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "7. lscpu" >> $OUTPUT_FILE -eval "lscpu" >> $OUTPUT_FILE +echo "7. lscpu" >> "${OUTPUT_FILE}" +eval "lscpu" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "8. numactl --hardware" >> $OUTPUT_FILE +echo "8. numactl --hardware" >> "${OUTPUT_FILE}" if [[ ${MLC_SUDO_USER} == "yes" ]]; then echo "${MLC_SUDO} numactl --hardware" - eval "${MLC_SUDO} numactl --hardware" >> $OUTPUT_FILE + eval "${MLC_SUDO} numactl --hardware" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? else - echo "Requires SUDO permission" >> $OUTPUT_FILE + echo "Requires SUDO permission" >> "${OUTPUT_FILE}" fi -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "9. /proc/meminfo" >> $OUTPUT_FILE -eval "cat /proc/meminfo" >> $OUTPUT_FILE +echo "9. /proc/meminfo" >> "${OUTPUT_FILE}" +eval "cat /proc/meminfo" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "10. who -r" >> $OUTPUT_FILE -eval "who -r" >> $OUTPUT_FILE +echo "10. who -r" >> "${OUTPUT_FILE}" +eval "who -r" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "11. Systemd service manager version" >> $OUTPUT_FILE -eval "systemctl --version | head -n 1" >> $OUTPUT_FILE +echo "11. Systemd service manager version" >> "${OUTPUT_FILE}" +eval "systemctl --version | head -n 1" >> "${OUTPUT_FILE}" #test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE -eval "systemctl list-unit-files" >> $OUTPUT_FILE +echo "12. Services, from systemctl list-unit-files" >> "${OUTPUT_FILE}" +eval "systemctl list-unit-files" >> "${OUTPUT_FILE}" #test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE -eval "cat /proc/cmdline" >> $OUTPUT_FILE +echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> "${OUTPUT_FILE}" +eval "cat /proc/cmdline" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "14. cpupower frequency-info" >> $OUTPUT_FILE -eval "cpupower frequency-info" >> $OUTPUT_FILE -test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> $OUTPUT_FILE -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "14. cpupower frequency-info" >> "${OUTPUT_FILE}" +eval "cpupower frequency-info" >> "${OUTPUT_FILE}" +test $? -eq 0 || echo "FAILED: cpupower frequency-info" >> "${OUTPUT_FILE}" +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "15. sysctl" >> $OUTPUT_FILE +echo "15. sysctl" >> "${OUTPUT_FILE}" if [[ ${MLC_SUDO_USER} == "yes" ]]; then echo "${MLC_SUDO} sysctl -a" - eval "${MLC_SUDO} sysctl -a" >> $OUTPUT_FILE + eval "${MLC_SUDO} sysctl -a" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? else - echo "Requires SUDO permission" >> $OUTPUT_FILE + echo "Requires SUDO permission" >> "${OUTPUT_FILE}" fi -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "16. /sys/kernel/mm/transparent_hugepage" >> $OUTPUT_FILE -eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> $OUTPUT_FILE +echo "16. /sys/kernel/mm/transparent_hugepage" >> "${OUTPUT_FILE}" +eval "cat /sys/kernel/mm/transparent_hugepage/enabled" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> $OUTPUT_FILE -eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> $OUTPUT_FILE +echo "17. /sys/kernel/mm/transparent_hugepage/khugepaged" >> "${OUTPUT_FILE}" +eval "cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "18. OS release" >> $OUTPUT_FILE -eval "cat /etc/os-release" >> $OUTPUT_FILE +echo "18. OS release" >> "${OUTPUT_FILE}" +eval "cat /etc/os-release" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "19. Disk information" >> $OUTPUT_FILE -eval "lsblk" >> $OUTPUT_FILE +echo "19. Disk information" >> "${OUTPUT_FILE}" +eval "lsblk" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "20. /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE -eval "ls /sys/devices/virtual/dmi/id" >> $OUTPUT_FILE +echo "20. /sys/devices/virtual/dmi/id" >> "${OUTPUT_FILE}" +eval "ls /sys/devices/virtual/dmi/id" >> "${OUTPUT_FILE}" test $? -eq 0 || exit $? -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "21. dmidecode" >> $OUTPUT_FILE +echo "21. dmidecode" >> "${OUTPUT_FILE}" if [[ ${MLC_SUDO_USER} == "yes" ]]; then - eval "${MLC_SUDO} dmidecode" >> $OUTPUT_FILE - test $? -eq 0 || echo "FAILED: dmidecode" >> $OUTPUT_FILE + eval "${MLC_SUDO} dmidecode" >> "${OUTPUT_FILE}" + test $? -eq 0 || echo "FAILED: dmidecode" >> "${OUTPUT_FILE}" else - echo "Requires SUDO permission" >> $OUTPUT_FILE + echo "Requires SUDO permission" >> "${OUTPUT_FILE}" fi -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "22. BIOS" >> $OUTPUT_FILE +echo "22. BIOS" >> "${OUTPUT_FILE}" if [[ ${MLC_SUDO_USER} == "yes" ]]; then - eval "${MLC_SUDO} dmidecode -t bios" >> $OUTPUT_FILE - test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> $OUTPUT_FILE + eval "${MLC_SUDO} dmidecode -t bios" >> "${OUTPUT_FILE}" + test $? -eq 0 || echo "FAILED: dmidecode -t bios" >> "${OUTPUT_FILE}" else - echo "Requires SUDO permission" >> $OUTPUT_FILE + echo "Requires SUDO permission" >> "${OUTPUT_FILE}" fi -echo "------------------------------------------------------------" >> $OUTPUT_FILE +echo "------------------------------------------------------------" >> "${OUTPUT_FILE}" -echo "System information has been saved to $OUTPUT_FILE" +echo "System information has been saved to "${OUTPUT_FILE}""