From 9064344b059f924f2e83a085665c3bb8e3bdec5a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 26 Mar 2025 22:03:20 +0000 Subject: [PATCH 1/6] Fixes for aocc (#324) Co-authored-by: Arjun --- script/get-aocc/customize.py | 2 +- script/get-aocc/meta.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 354a34d80..5c8b33712 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -42,7 +42,7 @@ def preprocess(i): def detect_version(i): - r = i['automation'].parse_version({'match_text': r'AMD\s+clang\sversion\s([\d.]+)', + r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)', 'group_number': 1, 'env_key': 'MLC_AOCC_VERSION', 'which_env': i['env']}) diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index ff12c56ec..e1ffa999c 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -44,6 +44,6 @@ tags: - aocc uid: 1ceb0656e99a44ec variations: - _path.#: + path.#: env: - MLC_AOCC_DIR_PATH: # + MLC_AOCC_DIR_PATH: '#' From 4bb611bec3d4cdeb2fa5b2270637d9d1c16e7eae Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 10:51:59 +0000 Subject: [PATCH 2/6] Modularize dmidecode meminfo parsing (#325) * Fix a bug in script module not updating state in postprocess, modularized parsing of memory info --- .github/workflows/build_wheel.yml | 1 - automation/script/module.py | 1 + .../customize.py | 27 ++++++++++---- .../detect_memory.sh | 3 +- .../meta.yaml | 8 +++- .../parse-dmidecode-memory-info/customize.py | 37 +++++++++++++++++++ .../get_memory_info.py | 18 +++++++-- script/parse-dmidecode-memory-info/meta.yaml | 21 +++++++++++ script/parse-dmidecode-memory-info/run.bat | 23 ++++++++++++ script/parse-dmidecode-memory-info/run.sh | 17 +++++++++ 10 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 script/parse-dmidecode-memory-info/customize.py rename script/{get-mlperf-inference-sut-description => parse-dmidecode-memory-info}/get_memory_info.py (80%) create mode 100644 script/parse-dmidecode-memory-info/meta.yaml create mode 100644 script/parse-dmidecode-memory-info/run.bat create mode 100644 script/parse-dmidecode-memory-info/run.sh diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 5467a7cc9..3b7919325 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -8,7 +8,6 @@ on: - VERSION jobs: - build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/automation/script/module.py b/automation/script/module.py index db32015b4..1325f5197 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1860,6 +1860,7 @@ def _run(self, i): run_script_input['meta'] = meta run_script_input['env'] = env + run_script_input['state'] = state run_script_input['run_state'] = run_state run_script_input['recursion'] = recursion diff --git a/script/get-mlperf-inference-sut-description/customize.py b/script/get-mlperf-inference-sut-description/customize.py index 93746764e..879f98164 100644 --- a/script/get-mlperf-inference-sut-description/customize.py +++ b/script/get-mlperf-inference-sut-description/customize.py @@ -2,6 +2,7 @@ import os import json import shutil +from utils import * def preprocess(i): @@ -41,6 +42,8 @@ def preprocess(i): sut_desc_path = env['MLC_MLPERF_INFERENCE_SUT_DESC_PATH'] sut_path = os.path.join(sut_desc_path, "suts", sut + ".json") + env['MLC_SUT_PATH'] = sut_path + if os.path.exists(sut_path) and env.get('MLC_SUT_DESC_CACHE', '') == "yes": print(f"Reusing SUT description file {sut}") state['MLC_SUT_META'] = json.load(open(sut_path)) @@ -168,18 +171,11 @@ def preprocess(i): if env.get('MLC_SUDO_USER', '') == "yes" and env.get( 'MLC_HOST_OS_TYPE', 'linux'): + env['MLC_MEMINFO_FILE'] = os.path.join(os.getcwd(), 'meminfo.dump') r = i['automation'].run_native_script( {'run_script_input': i['run_script_input'], 'env': env, 'script_name': 'detect_memory'}) if r['return'] > 0: return r - if env.get('MLC_HOST_MEM_INFO', '') != '': - state['MLC_SUT_META']['host_memory_configuration'] = env['MLC_HOST_MEM_INFO'] - - state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items())) - - sut_file = open(sut_path, "w") - json.dump(state['MLC_SUT_META'], sut_file, indent=4) - sut_file.close() return {'return': 0} @@ -187,5 +183,20 @@ def preprocess(i): def postprocess(i): env = i['env'] + state = i['state'] + + if env.get('MLC_MEMINFO_OUTFILE', '') != '' and os.path.exists( + env['MLC_MEMINFO_OUTFILE']): + with open(env['MLC_MEMINFO_OUTFILE'], "r") as f: + data = f.read() + state['MLC_SUT_META']['host_memory_configuration'] = data + + state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items())) + + sut_path = env['MLC_SUT_PATH'] + + sut_file = open(sut_path, "w") + json.dump(state['MLC_SUT_META'], sut_file, indent=4) + sut_file.close() return {'return': 0} diff --git a/script/get-mlperf-inference-sut-description/detect_memory.sh b/script/get-mlperf-inference-sut-description/detect_memory.sh index 4a21653b9..3d53fba78 100644 --- a/script/get-mlperf-inference-sut-description/detect_memory.sh +++ b/script/get-mlperf-inference-sut-description/detect_memory.sh @@ -1,7 +1,6 @@ #!/bin/bash if [[ ${MLC_SUDO_USER} == "yes" ]]; then - ${MLC_SUDO} dmidecode -t memory > meminfo.out - ${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/get_memory_info.py + ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE} fi test $? -eq 0 || return $? diff --git a/script/get-mlperf-inference-sut-description/meta.yaml b/script/get-mlperf-inference-sut-description/meta.yaml index c46bf930b..5e935a0dc 100644 --- a/script/get-mlperf-inference-sut-description/meta.yaml +++ b/script/get-mlperf-inference-sut-description/meta.yaml @@ -29,16 +29,22 @@ deps: MLC_HOST_OS_TYPE: - linux tags: detect,sudo -- tags: get,generic-python-lib,_package.dmiparser - env: MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH extra_cache_tags: mlperf,inference,sut,descriptions tags: get,cache,dir,_name.mlperf-inference-sut-descriptions + +posthook_deps: + - tags: parse,dmidecode,memory,info + enable_if_env: + MLC_MEMINFO_FILE: + - on docker: run: false input_mapping: name: MLC_HW_NAME submitter: MLC_MLPERF_SUBMITTER + memory: MLC_DETERMINE_MEMORY_CONFIGURATION new_env_keys: - MLC_HW_* - MLC_SUT_* diff --git a/script/parse-dmidecode-memory-info/customize.py b/script/parse-dmidecode-memory-info/customize.py new file mode 100644 index 000000000..b5a1805f7 --- /dev/null +++ b/script/parse-dmidecode-memory-info/customize.py @@ -0,0 +1,37 @@ +from mlc import utils +import os +import subprocess + + +def preprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + input_file = env.get('MLC_MEMINFO_FILE', '') + + if input_file == '': + return {'return': 1, 'error': 'Please provide a valid input file containing the meminfo output from dmidecode -t memory'} + + output_file = env.get( + 'MLC_MEMINFO_OUTFILE', + os.path.join( + os.getcwd(), + 'meminfo.txt')) + + env['MLC_MEMINFO_OUTFILE'] = output_file + + env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'], "get_memory_info.py")} {input_file} {output_file}""" + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + return {'return': 0} diff --git a/script/get-mlperf-inference-sut-description/get_memory_info.py b/script/parse-dmidecode-memory-info/get_memory_info.py similarity index 80% rename from script/get-mlperf-inference-sut-description/get_memory_info.py rename to script/parse-dmidecode-memory-info/get_memory_info.py index 220517576..55c521a60 100644 --- a/script/get-mlperf-inference-sut-description/get_memory_info.py +++ b/script/parse-dmidecode-memory-info/get_memory_info.py @@ -1,8 +1,20 @@ import os import json +import argparse from dmiparser import DmiParser -with open("meminfo.out", "r") as f: +# Adding argument parser +parser = argparse.ArgumentParser( + description='Process meminfo and output MLC_HOST_MEM_INFO.') +parser.add_argument( + "input_file", + help="Path to the input file (e.g. meminfo.out)") +parser.add_argument( + "output_file", + help="Path to the output file (e.g. tmp-run-env.out)") +args = parser.parse_args() + +with open(args.input_file, "r") as f: text = f.read() parser = DmiParser(text, sort_keys=True, indent=4) @@ -57,5 +69,5 @@ meminfo.append("; ".join(item['info'])) meminfo_string = ", ".join(meminfo) - with open("tmp-run-env.out", "w") as f: - f.write(f"MLC_HOST_MEM_INFO={meminfo_string}") + with open(args.output_file, "w") as f: + f.write(meminfo_string) diff --git a/script/parse-dmidecode-memory-info/meta.yaml b/script/parse-dmidecode-memory-info/meta.yaml new file mode 100644 index 000000000..0f4c1b8e4 --- /dev/null +++ b/script/parse-dmidecode-memory-info/meta.yaml @@ -0,0 +1,21 @@ +alias: parse-dmidecode-memory-info +automation_alias: script +automation_uid: 5b4e0237da074764 +category: MLC Utils +deps: + - tags: get,generic-python-lib,_package.dmiparser +new_env_keys: + - MLC_MEMINFO_OUTFILE +input_mapping: + input: MLC_MEMINFO_FILE + output: MLC_MEMINFO_OUTFILE +new_state_keys: [] +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- dmidecode +- parse +- memory +- info +uid: 1a6be59b1e564fbf diff --git a/script/parse-dmidecode-memory-info/run.bat b/script/parse-dmidecode-memory-info/run.bat new file mode 100644 index 000000000..12c8a6224 --- /dev/null +++ b/script/parse-dmidecode-memory-info/run.bat @@ -0,0 +1,23 @@ +@echo off +setlocal enabledelayedexpansion + +:: Function to exit if the last command failed +:exit_if_error +if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL% +exit /b 0 + +:: Function to run a command +:run +echo Running: +echo %1 +echo. + +if /I "%MLC_FAKE_RUN%" NEQ "yes" ( + call %1 + call :exit_if_error +) +exit /b 0 + +:: Add your run commands here... +call :run "%MLC_RUN_CMD%" + diff --git a/script/parse-dmidecode-memory-info/run.sh b/script/parse-dmidecode-memory-info/run.sh new file mode 100644 index 000000000..c4542b8c2 --- /dev/null +++ b/script/parse-dmidecode-memory-info/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +function exit_if_error() { + test $? -eq 0 || exit $? +} + +function run() { + echo "Running: " + echo "$1" + echo "" + if [[ ${MLC_FAKE_RUN} != 'yes' ]]; then + eval "$1" + exit_if_error + fi +} + +#Add your run commands here... +run "$MLC_RUN_CMD" From 6936554e4671b935d0028f52d51f802c65d3e411 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:26:04 +0000 Subject: [PATCH 3/6] Support aocc download and install (#326) * Support aocc download with EULA * Fix get-gcc,get-oneapi variations --- .github/workflows/build_wheel.yml | 1 + automation/script/module.py | 3 +++ script/download-file/meta.yaml | 3 +++ script/extract-file/customize.py | 3 ++- script/extract-file/meta.yaml | 3 +++ script/get-aocc/customize.py | 7 +++++++ script/get-aocc/meta.yaml | 22 ++++++++++++++++++++++ script/get-gcc/meta.yaml | 4 ++-- script/get-one-api/meta.yaml | 2 +- 9 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 3b7919325..5467a7cc9 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -8,6 +8,7 @@ on: - VERSION jobs: + build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/automation/script/module.py b/automation/script/module.py index 1325f5197..02db3571c 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4727,6 +4727,9 @@ def find_cached_script(i): r = docker_utils.get_container_path_script(i) if not os.path.exists(r['value_env']): # Need to rm this cache entry + logger.debug( + recursion_spaces + + ' - Skipping cached entry as the dependent path {} is missing!'.format(r['value_env'])) skip_cached_script = True continue diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index 832275c1c..6f940186e 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -6,6 +6,7 @@ can_force_cache: true category: DevOps automation default_env: MLC_RCLONE_COPY_USING: sync + MLC_EXTRACT_REMOVE_EXTRACTED: no deps: - tags: detect,os - enable_if_env: @@ -40,6 +41,8 @@ tags_help: download file uid: 9cdc8dc41aae437e variations: cmutil: + alias: mlcutil + mlcutil: default: true env: MLC_DOWNLOAD_TOOL: cmutil diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index d55f8787c..c7e50be3a 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -1,6 +1,7 @@ from mlc import utils import os import hashlib +from utils import * def preprocess(i): @@ -209,7 +210,7 @@ def postprocess(i): env['MLC_GET_DEPENDENT_CACHED_PATH'] = filepath # Check if need to remove archive after extraction - if env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '').lower() != 'no': + if is_true(env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '')): archive_filepath = env.get('MLC_EXTRACT_FILEPATH', '') if archive_filepath != '' and os.path.isfile(archive_filepath): os.remove(archive_filepath) diff --git a/script/extract-file/meta.yaml b/script/extract-file/meta.yaml index 56f29fe1d..7e69437e1 100644 --- a/script/extract-file/meta.yaml +++ b/script/extract-file/meta.yaml @@ -36,9 +36,12 @@ tags_help: extract file uid: 3f0b76219d004817 variations: keep: + group: keep + default: true env: MLC_EXTRACT_REMOVE_EXTRACTED: 'no' no-remove-extracted: + group: keep env: MLC_EXTRACT_REMOVE_EXTRACTED: 'no' path.#: diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 5c8b33712..1e42e360a 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -25,6 +25,13 @@ def preprocess(i): aocc_path = env['MLC_AOCC_DIR_PATH'] if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')): env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') + else: + for l in os.listdir(aocc_path): + if os.path.exists(os.path.join( + aocc_path, l, 'bin', 'clang')): + aocc_path = os.path.join(aocc_path, l) + env['MLC_AOCC_DIR_PATH'] = aocc_path + env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') r = i['automation'].find_artifact({'file_name': exe_c, 'env': env, diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index e1ffa999c..41ff05210 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -6,10 +6,24 @@ category: Compiler automation clean_files: [] deps: - tags: detect,os +- tags: download,file,_mlcutil + extra_cache_tags: aocc,download + update_tags_from_env_with_prefix: + _url.: + - MLC_AOCC_URL + force_cache: true + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_AOCC_TAR_FILE_PATH + enable_if_env: + MLC_AOCC_URL: + - on + MLC_AOCC_ACCEPT_EULA: + - on - tags: extract,file update_tags_from_env_with_prefix: _path.: - MLC_AOCC_TAR_FILE_PATH + extra_cache_tags: aocc,extract force_cache: true env: MLC_EXTRACT_FINAL_ENV_NAME: MLC_AOCC_DIR_PATH @@ -21,6 +35,7 @@ deps: input_mapping: tar_file_path: MLC_AOCC_TAR_FILE_PATH aocc_dir: MLC_AOCC_DIR_PATH + accept_eula: MLC_AOCC_ACCEPT_EULA name: Detect or install AOCC compiler new_env_keys: @@ -43,6 +58,13 @@ tags: - get - aocc uid: 1ceb0656e99a44ec +versions: + 5.0.0: + env: + MLC_AOCC_URL: https://download.amd.com/developer/eula/aocc/aocc-5-0/aocc-compiler-5.0.0.tar + MLC_DOWNLOAD_CHECKSUM: 966fac2d2c759e9de6e969c10ada7a7b306c113f7f1e07ea376829ec86380daa + MLC_AOCC_NEEDS_TAR: yes + MLC_VERSION: '5.0.0-Build#1377' variations: path.#: env: diff --git a/script/get-gcc/meta.yaml b/script/get-gcc/meta.yaml index 4de1812f5..d8d7827b3 100644 --- a/script/get-gcc/meta.yaml +++ b/script/get-gcc/meta.yaml @@ -31,6 +31,6 @@ tags: - get-gcc uid: dbf4ab5cbed74372 variations: - _path.#: + path.#: env: - MLC_GCC_DIR_PATH: # + MLC_GCC_DIR_PATH: '#' diff --git a/script/get-one-api/meta.yaml b/script/get-one-api/meta.yaml index 1033eb772..47cf218b0 100644 --- a/script/get-one-api/meta.yaml +++ b/script/get-one-api/meta.yaml @@ -26,6 +26,6 @@ tags: - get-oneapi uid: 1af872e81ef54742 variations: - _path.#: + path.#: env: MLC_ONEAPI_DIR_PATH: "#" From 3db9d328eb6e20766c8b13d554c8a64aefd1f65e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:35:49 +0000 Subject: [PATCH 4/6] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7dea76edb..6d7de6e6a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.0.2 From 2aead77795ac36dd8828275841e1256031de5319 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:39:09 +0000 Subject: [PATCH 5/6] Update and rename build_wheel.yml to build_wheel_off.yml --- .github/workflows/{build_wheel.yml => build_wheel_off.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build_wheel.yml => build_wheel_off.yml} (98%) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel_off.yml similarity index 98% rename from .github/workflows/build_wheel.yml rename to .github/workflows/build_wheel_off.yml index 5467a7cc9..a24482b1c 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel_off.yml @@ -1,4 +1,4 @@ -name: Build wheel and release into PYPI +name: Build wheel and release into PYPI (off now) on: push: From d3fd12ab0244e1cbdb7a0b9e093cd05d6b4408dc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:39:37 +0000 Subject: [PATCH 6/6] Update and rename build_wheels_on_release.yml to build_wheels.yml --- .../workflows/{build_wheels_on_release.yml => build_wheels.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build_wheels_on_release.yml => build_wheels.yml} (95%) diff --git a/.github/workflows/build_wheels_on_release.yml b/.github/workflows/build_wheels.yml similarity index 95% rename from .github/workflows/build_wheels_on_release.yml rename to .github/workflows/build_wheels.yml index fd62c80c2..708ae35e4 100644 --- a/.github/workflows/build_wheels_on_release.yml +++ b/.github/workflows/build_wheels.yml @@ -1,4 +1,4 @@ -name: Build wheel and release into PYPI +name: Build wheel (only src) and release into PYPI on: release: