diff --git a/VERSION b/VERSION index 1750564f2..d169b2f2d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.6 +0.0.8 diff --git a/automation/script/docker.py b/automation/script/docker.py index 9852937c1..bebee59e1 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -15,6 +15,8 @@ def dockerfile(self_module, input_params): if prune_result['return'] > 0: return prune_result + logger = self_module.logger + run_command_arc = prune_result['new_input'] current_directory = os.getcwd() is_quiet_mode = input_params.get('quiet', False) @@ -99,7 +101,7 @@ def dockerfile(self_module, input_params): if not docker_settings.get('run', True) and not input_params.get( 'docker_run_override', False): - logging.info("Docker 'run' is set to False in meta.json") + logger.info("Docker 'run' is set to False in meta.json") continue # Handle build dependencies @@ -186,7 +188,7 @@ def dockerfile(self_module, input_params): if dockerfile_result['return'] > 0: return dockerfile_result - logging.info(f"Dockerfile generated at {dockerfile_path}") + logger.info(f"Dockerfile generated at {dockerfile_path}") return {'return': 0} @@ -209,7 +211,7 @@ def docker_run(self_module, i): quiet = i.get('quiet', False) verbose = i.get('v', False) show_time = i.get('show_time', False) - + logger = self_module.logger env = i.get('env', {}) regenerate_docker_file = not i.get('docker_noregenerate', False) @@ -325,7 +327,7 @@ def docker_run(self_module, i): # Skip scripts marked as non-runnable if not docker_settings.get('run', True) and not i.get( 'docker_run_override', False): - logging.info("docker.run set to False in meta.yaml") + logger.info("docker.run set to False in meta.yaml") continue r = self_module._update_env_from_input(env, i) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 471b494ef..75c1f68a4 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -1,7 +1,6 @@ import os from mlc import utils from utils import * -import logging from pathlib import PureWindowsPath, PurePosixPath from script.docker_utils import * import copy diff --git a/automation/script/module.py b/automation/script/module.py index ae8e3acd7..cbb3bcb58 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -26,7 +26,6 @@ class ScriptAutomation(Automation): ############################################################ def __init__(self, action_object, automation_file): super().__init__(action_object, "script", automation_file) - # logger.basicConfig(level=logger.INFO) self.os_info = {} self.run_state = {} self.run_state['deps'] = [] @@ -35,6 +34,10 @@ def __init__(self, action_object, automation_file): self.run_state['version_info'] = [] self.run_state['cache'] = False self.file_with_cached_state = 'mlc-cached-state.json' + # self.logger = logging.getLogger() + # logging.basicConfig(level=logging.INFO) + self.logger = self.action_object.logger + self.logger.propagate = False self.tmp_file_env = 'tmp-env' self.tmp_file_env_all = 'tmp-env-all' @@ -188,7 +191,7 @@ def run(self, i): (print_readme) (bool): if True, will print README with all MLC steps (deps) to run a given script - (script_call_prefix) (str): how to call script in logs and READMEs (mlc run script) + (script_call_prefix) (str): how to call script in logs and READMEs (mlcr) (skip_sys_utils) (bool): if True, set env['MLC_SKIP_SYS_UTILS']='yes' to skip MLC sys installation @@ -233,7 +236,7 @@ def _run(self, i): repro = i.get('repro', False) repro_prefix = '' - logger = self.action_object.logger + logger = self.logger if repro: repro_prefix = i.get('repro_prefix', '') @@ -539,15 +542,15 @@ def _run(self, i): mlc_script_info = i.get('script_call_prefix', '').strip() if mlc_script_info == '': - mlc_script_info = 'mlc run script' + mlc_script_info = 'mlcr ' if not mlc_script_info.endswith(' '): mlc_script_info += ' ' - x = '--tags=' + x = '' y = ',' if parsed_script_alias != '': mlc_script_info += parsed_script_alias - x = '--tags="' + x = '"' if len(script_tags) > 0 or len(variation_tags) > 0: mlc_script_info += x @@ -3800,7 +3803,7 @@ def _get_readme(self, cmd_parts, run_state): ```bash """ - cmd = "mlc run script " + cmd = "mlcr " for cmd_part in cmd_parts: x = '"' if ' ' in cmd_part and not cmd_part.startswith('-') else '' @@ -3823,7 +3826,7 @@ def _get_readme(self, cmd_parts, run_state): xversion = ' --version={}\n'.format(version) content += "```bash\n" - content += "mlc run script --tags=" + \ + content += "mlcr " + \ dep_tags + "{}\n".format(xversion) content += "```\n\n" @@ -3887,7 +3890,7 @@ def _get_deps_run_cmds(self, deps): run_cmds = [] for dep_tags in deps: - run_cmds.append("mlc run script --tags=" + dep_tags) + run_cmds.append("mlcr " + dep_tags) return run_cmds @@ -6164,7 +6167,8 @@ def select_script_item(lst, text, recursion_spaces, ' - More than 1 {} found for "{}":'.format(text, script_tags_string) if not logger: - logger = logging.getLoger() + return {'return': 1, 'error': 'No logger provided'} + # If quiet, select 0 (can be sorted for determinism) if quiet: logger.debug(string1) diff --git a/git_commit_hash.txt b/git_commit_hash.txt index d5693f79a..d21d33c55 100644 --- a/git_commit_hash.txt +++ b/git_commit_hash.txt @@ -1 +1 @@ -90fbda0c7079a0a09688aa2045d2f571ef61c222 +30a1c0c5227d564cf472bbc128a6d1919f29ad25 diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index fbe70dde4..f94dfa86b 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -5,7 +5,6 @@ import shutil import subprocess import copy -import mlc import platform import sys import mlperf_utils @@ -354,8 +353,14 @@ def postprocess(i): "os_version": platform.platform(), "cpu_version": platform.processor(), "python_version": sys.version, - "mlc_version": mlc.__version__ } + try: + import importlib.metadata + mlc_version = importlib.metadata.version("mlc") + host_info["mlc_version"] = mlc_version + except Exception as e: + error = format(e) + mlc_version = "unknown" x = '' if env.get('MLC_HOST_OS_FLAVOR', '') != '': @@ -371,6 +376,7 @@ def postprocess(i): # Check CM automation repository repo_name = 'mlcommons@mlperf-automations' repo_hash = '' + mlc = i['automation'].action_object r = mlc.access({'action': 'find', 'automation': 'repo', 'item': 'mlcommons@mlperf-automations,9e97bb72b0474657'}) if r['return'] == 0 and len(r['list']) == 1: @@ -405,7 +411,7 @@ def postprocess(i): readme_init = "*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLC version: {}\n\n".format(platform.platform(), - platform.processor(), sys.version, mlc.__version__) + platform.processor(), sys.version, mlc_version) x = repo_name if repo_hash != '': @@ -629,7 +635,7 @@ def postprocess(i): 'new_env', os.path.join( output_dir, - "os_info.json")) + "os_info.json"), mlc) dump_script_output( "detect,cpu", env, @@ -637,7 +643,7 @@ def postprocess(i): 'new_env', os.path.join( output_dir, - "cpu_info.json")) + "cpu_info.json"), mlc) env['MLC_DUMP_RAW_PIP_FREEZE_FILE_PATH'] = os.path.join( env['MLC_MLPERF_OUTPUT_DIR'], "pip_freeze.raw") dump_script_output( @@ -647,12 +653,12 @@ def postprocess(i): 'new_state', os.path.join( output_dir, - "pip_freeze.json")) + "pip_freeze.json"), mlc) return {'return': 0} -def dump_script_output(script_tags, env, state, output_key, dump_file): +def dump_script_output(script_tags, env, state, output_key, dump_file, mlc): mlc_input = {'action': 'run', 'automation': 'script', diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 384b0c9b8..44ca20167 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -2,7 +2,6 @@ import os import json import shutil -import mlc import sys from tabulate import tabulate import mlperf_utils @@ -700,6 +699,7 @@ def generate_submission(env, state, inp, submission_division): 'env': {'MLC_PLATFORM_DETAILS_FILE_PATH': os.path.join(measurement_path, "system_info.txt")}, 'quiet': True } + mlc = i['automation'].action_object r = mlc.access(mlc_input) if r['return'] > 0: return r diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 99f07fa14..9277ab7f0 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -248,7 +248,7 @@ def postprocess(i): CONTAINER = f"""{env['MLC_CONTAINER_TOOL']} run -dt {run_opts} --rm {docker_image_repo}/{docker_image_name}:{docker_image_tag} bash""" CMD = f"""ID=`{CONTAINER}` && {env['MLC_CONTAINER_TOOL']} exec $ID bash -c '{run_cmd}'""" - if not is_true(env.get('MLC_KEEP_DETACHED_CONTAINER', '')): + if is_true(env.get('MLC_KILL_DETACHED_CONTAINER', False)): CMD += f""" && {env['MLC_CONTAINER_TOOL']} kill $ID >/dev/null""" CMD += ' && echo "ID=$ID"' diff --git a/script/run-terraform/customize.py b/script/run-terraform/customize.py index 5728ce687..15687e6a9 100644 --- a/script/run-terraform/customize.py +++ b/script/run-terraform/customize.py @@ -1,5 +1,4 @@ from mlc import utils -import mlc import os import shutil import json @@ -72,6 +71,7 @@ def postprocess(i): cmd = cmd.replace(":", "=") cmd = cmd.replace(";;", ",") run_input['run_cmds'].append(cmd) + mlc = i['automation'].action_object r = mlc.access(run_input) if r['return'] > 0: return r diff --git a/setup.py b/setup.py index 5a18472c6..0ba25f32d 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def run(self): 'target': 'repo', 'repo': 'mlcommons@mlperf-automations', 'branch': branch, - 'checkout': commit_hash + # 'checkout': commit_hash }) print(res) if res['return'] > 0: