From e74e26e6ab5589059a0a861d0920065a1caca72b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 23 Jan 2025 23:23:54 +0530 Subject: [PATCH 01/14] Fixes for MLC docker run --- automation/script/docker.py | 7 +++++-- script/build-dockerfile/customize.py | 4 ++-- script/run-mlperf-inference-app/customize.py | 6 +++++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 99fa619cf..58339632e 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -22,7 +22,7 @@ def dockerfile(self_module, input_params): is_console_output = input_params.get('out') == 'con' # Step 2: Search for scripts - search_result = self_module.search(input_params) + search_result = self_module.search(input_params.copy()) if search_result['return'] > 0: return search_result @@ -190,7 +190,10 @@ def dockerfile(self_module, input_params): } mlc_docker_input.update(dockerfile_inputs) + #print(mlc_docker_input) + #return {'return': 1} dockerfile_result = self_module.action_object.access(mlc_docker_input) + #dockerfile_result = dockerfile(self_module,mlc_docker_input) if dockerfile_result['return'] > 0: return dockerfile_result @@ -241,7 +244,7 @@ def docker_run(self_module, i): # Save current directory and prepare to search for scripts cur_dir = os.getcwd() - r = self_module.search(i) + r = self_module.search(i.copy()) if r['return'] > 0: return r diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 76c184782..2bf25ed22 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -30,12 +30,12 @@ def preprocess(i): if env.get('MLC_DOCKER_RUN_SCRIPT_TAGS', '') != '': script_tags = env['MLC_DOCKER_RUN_SCRIPT_TAGS'] found_scripts = automation.action_object.access( - {'action': 'search', 'automation': 'script', 'tags': script_tags}) + {'action': 'search', 'target': 'script', 'tags': script_tags}) scripts_list = found_scripts['list'] if not scripts_list: return {'return': 1, - 'error': 'No CM script found for tags ' + script_tags} + 'error': 'No automation script found for tags=' + script_tags} if len(scripts_list) > 1: return { diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 5bd17603e..84e7a131e 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -3,7 +3,7 @@ import json import shutil import subprocess -import mlc +#import mlc import copy import mlperf_utils @@ -276,6 +276,10 @@ def preprocess(i): for k in docker_extra_input: ii[k] = docker_extra_input[k] + mlc = i['automation'].action_object + + #print(ii) + #return {'return': 1} r = mlc.access(ii) if r['return'] > 0: return r From 89214ba10f50bbb3d29ea03c06160113e1214729 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 05:13:06 +0530 Subject: [PATCH 02/14] Fixes for mlc docker run --- automation/script/docker.py | 62 +++++++++++++-------------- automation/script/docker_utils.py | 53 ++++++++++++++--------- script/build-dockerfile/customize.py | 2 +- script/build-dockerfile/meta.yaml | 3 ++ script/get-sys-utils-cm/meta.yaml | 1 + script/run-docker-container/meta.yaml | 3 ++ 6 files changed, 71 insertions(+), 53 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 58339632e..433d9cb24 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -133,32 +133,9 @@ def dockerfile(self_module, input_params): run_command_string = regenerate_result['run_cmd_string'] - # Collect Dockerfile inputs - dockerfile_inputs = { - key: input_params.get( - f"docker_{key}", docker_settings.get( - key, get_docker_default(key))) - for key in [ - "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", - "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", - "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" - ] - if (value := input_params.get(f"docker_{key}", docker_settings.get(key, get_docker_default(key)))) is not None - } - - # Determine Dockerfile suffix and path - docker_base_image = dockerfile_inputs.get('base_image') - docker_path = dockerfile_inputs.get('path') - if not docker_path: - docker_path = script_directory - docker_filename_suffix = ( - docker_base_image.replace('/', '-').replace(':', '-') - if docker_base_image else f"{dockerfile_inputs['os']}_{dockerfile_inputs['os_version']}" - ) - dockerfile_path = os.path.join( - docker_path, - 'dockerfiles', - f"{docker_filename_suffix}.Dockerfile") + # Prepare Docker-specific inputs + docker_inputs, dockerfile_path = prepare_docker_inputs( + input_params, docker_settings, script_directory) # Handle optional dependencies and comments if input_params.get('print_deps'): @@ -182,18 +159,15 @@ def dockerfile(self_module, input_params): # Generate Dockerfile mlc_docker_input = { 'action': 'run', 'automation': 'script', 'tags': 'build,dockerfile', - 'fake_run_option': " " if dockerfile_inputs.get('real_run') else " --fake_run", + 'fake_run_option': " " if docker_inputs.get('real_run') else " --fake_run", 'comments': comments, 'run_cmd': f"{run_command_string} --quiet", 'script_tags': input_params.get('tags'), 'env': environment_vars, 'dockerfile_env': dockerfile_environment_vars, 'quiet': True, 'v': input_params.get('v', False), 'real_run': True } - mlc_docker_input.update(dockerfile_inputs) + mlc_docker_input.update(docker_inputs) - #print(mlc_docker_input) - #return {'return': 1} dockerfile_result = self_module.action_object.access(mlc_docker_input) - #dockerfile_result = dockerfile(self_module,mlc_docker_input) if dockerfile_result['return'] > 0: return dockerfile_result @@ -256,7 +230,7 @@ def docker_run(self_module, i): state, const, const_state = i.get( 'state', {}), i.get( 'const', {}), i.get( - 'const_state', {}) + 'const_state', {}) variation_tags = [t[1:] for t in i.get('tags', '').split(",") if t.startswith("_")] @@ -313,6 +287,29 @@ def docker_run(self_module, i): if r['return'] > 0: return r + docker_settings = state['docker'] + + deps = docker_settings.get('deps', []) + if deps: + r = self_module._run_deps( + deps, [], env, {}, {}, {}, {}, '', [], '', False, '', verbose, + show_time, ' ', run_state) + if r['return'] > 0: + return r + + # For updating meta from update_meta_if_env + r = self_module.update_state_from_meta( + meta, env, state, const, const_state, deps=[], + post_deps=[], + prehook_deps=[], + posthook_deps=[], + new_env_keys=[], + new_state_keys=[], + run_state=run_state, + i=i) + if r['return'] > 0: + return r + # Skip scripts marked as non-runnable if not docker_settings.get('run', True) and not i.get( 'docker_run_override', False): @@ -331,6 +328,7 @@ def docker_run(self_module, i): # Prepare Docker-specific inputs docker_inputs, dockerfile_path = prepare_docker_inputs( i, docker_settings, script_path) + if docker_inputs is None: return {'return': 1, 'error': 'Error preparing Docker inputs'} diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 2e1f40b9f..228cc6902 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -37,7 +37,7 @@ def process_mounts(mounts, env, i, docker_settings): return None -def prepare_docker_inputs(i, docker_settings, script_path): +def prepare_docker_inputs(input_params, docker_settings, script_path): """ Prepares Docker-specific inputs such as Dockerfile path and runtime options. @@ -49,23 +49,34 @@ def prepare_docker_inputs(i, docker_settings, script_path): Returns: Tuple with Docker inputs dictionary and Dockerfile path or None in case of an error. """ - try: - dockerfile_path = os.path.join( - script_path, docker_settings.get( - 'dockerfile', 'Dockerfile')) - docker_args = docker_settings.get('args', {}) - docker_image = i.get('docker_image', docker_settings.get('image', '')) - - docker_inputs = { - 'dockerfile': dockerfile_path, - 'docker_image': docker_image, - 'docker_args': docker_args - } - - return docker_inputs, dockerfile_path - except Exception as e: - logging.error(f"Error preparing Docker inputs: {e}") - return None, None + # Collect Dockerfile inputs + docker_inputs = { + key: input_params.get( + f"docker_{key}", docker_settings.get( + key, get_docker_default(key))) + for key in [ + "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", + "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", + "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" + ] + if (value := input_params.get(f"docker_{key}", docker_settings.get(key, get_docker_default(key)))) is not None + } + + # Determine Dockerfile suffix and path + docker_base_image = docker_inputs.get('base_image') + docker_path = docker_inputs.get('path') + if not docker_path: + docker_path = script_path + docker_filename_suffix = ( + docker_base_image.replace('/', '-').replace(':', '-') + if docker_base_image else f"{docker_inputs['os']}_{docker_inputs['os_version']}" + ) + dockerfile_path = os.path.join( + docker_path, + 'dockerfiles', + f"{docker_filename_suffix}.Dockerfile") + + return docker_inputs, dockerfile_path def update_docker_paths(path, mounts=None, force_target_path=''): @@ -101,7 +112,9 @@ def update_docker_paths(path, mounts=None, force_target_path=''): # Determine the mount string based on whether the path is a file or # directory. if os.path.isfile(host_path) or not os.path.isdir(host_path): - mount_entry = f"{os.path.dirname(host_path)}:{os.path.dirname(container_path)}" + mount_entry = f"{ + os.path.dirname(host_path)}:{ + os.path.dirname(container_path)}" else: mount_entry = f"{host_path}:{container_path}" @@ -157,7 +170,7 @@ def regenerate_script_cmd(i): docker_run_cmd_prefix = i.get('docker_run_cmd_prefix', '') # Regenerate command from dictionary input - run_cmd = 'mlc run script' + run_cmd = 'mlcr' skip_input_for_fake_run = docker_settings.get( 'skip_input_for_fake_run', []) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 2bf25ed22..491c86976 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -343,7 +343,7 @@ def preprocess(i): if str(env.get('MLC_DOCKER_SKIP_MLC_SYS_UPGRADE', False) ).lower() not in ["true", "1", "yes"]: f.write(EOL + '# Install all system dependencies' + EOL) - f.write('RUN mlc run script --tags=get,sys-utils-cm --quiet' + EOL) + f.write('RUN mlc run script --tags=get,sys-utils-mlc --quiet' + EOL) if 'MLC_DOCKER_PRE_RUN_COMMANDS' in env: for pre_run_cmd in env['MLC_DOCKER_PRE_RUN_COMMANDS']: diff --git a/script/build-dockerfile/meta.yaml b/script/build-dockerfile/meta.yaml index 6dd1ee499..e6237fb6b 100644 --- a/script/build-dockerfile/meta.yaml +++ b/script/build-dockerfile/meta.yaml @@ -30,6 +30,9 @@ input_mapping: mlc_repo_branch: MLC_MLOPS_REPO_BRANCH comments: MLC_DOCKER_RUN_COMMENTS copy_files: MLC_DOCKER_COPY_FILES + base_image: MLC_DOCKER_IMAGE_BASE + os: MLC_DOCKER_OS + os_version: MLC_DOCKER_OS_VERSION docker_base_image: MLC_DOCKER_IMAGE_BASE docker_os: MLC_DOCKER_OS docker_os_version: MLC_DOCKER_OS_VERSION diff --git a/script/get-sys-utils-cm/meta.yaml b/script/get-sys-utils-cm/meta.yaml index b1bfc03ed..83e39e9a4 100644 --- a/script/get-sys-utils-cm/meta.yaml +++ b/script/get-sys-utils-cm/meta.yaml @@ -25,6 +25,7 @@ new_env_keys: tags: - get - sys-utils-cm +- sys-utils-mlc variations: user: diff --git a/script/run-docker-container/meta.yaml b/script/run-docker-container/meta.yaml index 394cd4c51..e18855026 100644 --- a/script/run-docker-container/meta.yaml +++ b/script/run-docker-container/meta.yaml @@ -29,9 +29,12 @@ input_mapping: device: MLC_DOCKER_ADD_DEVICE docker_image_base: MLC_DOCKER_IMAGE_BASE docker_base_image: MLC_DOCKER_IMAGE_BASE + base_image: MLC_DOCKER_IMAGE_BASE keep_detached: MLC_KEEP_DETACHED_CONTAINER docker_os: MLC_DOCKER_OS docker_os_version: MLC_DOCKER_OS_VERSION + os: MLC_DOCKER_OS + os_version: MLC_DOCKER_OS_VERSION extra_run_args: MLC_DOCKER_EXTRA_RUN_ARGS fake_run_option: MLC_DOCKER_FAKE_RUN_OPTION gh_token: MLC_GH_TOKEN From 66d5882177c44176a29edf2c3ad54ce54738566e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 23 Jan 2025 23:43:33 +0000 Subject: [PATCH 03/14] [Automated Commit] Format Codebase --- automation/script/docker_utils.py | 2 +- script/run-mlperf-inference-app/customize.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 228cc6902..f5b4ce14b 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -113,7 +113,7 @@ def update_docker_paths(path, mounts=None, force_target_path=''): # directory. if os.path.isfile(host_path) or not os.path.isdir(host_path): mount_entry = f"{ - os.path.dirname(host_path)}:{ + os.path.dirname(host_path)}: { os.path.dirname(container_path)}" else: mount_entry = f"{host_path}:{container_path}" diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 84e7a131e..a966adbc5 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -3,7 +3,7 @@ import json import shutil import subprocess -#import mlc +# import mlc import copy import mlperf_utils @@ -278,8 +278,8 @@ def preprocess(i): mlc = i['automation'].action_object - #print(ii) - #return {'return': 1} + # print(ii) + # return {'return': 1} r = mlc.access(ii) if r['return'] > 0: return r From 3462cc26a232dee1be89bd0f3cfe14afc1d78c07 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 05:43:32 +0530 Subject: [PATCH 04/14] Fixes to docker input options --- automation/script/docker.py | 14 ++++++++------ automation/script/docker_utils.py | 2 ++ script/build-dockerfile/customize.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 433d9cb24..fccac5eae 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -198,20 +198,22 @@ def docker_run(self_module, i): show_time = i.get('show_time', False) env = i.get('env', {}) - noregenerate_docker_file = i.get('docker_noregenerate', False) - norecreate_docker_image = not i.get('docker_recreate', False) + + regenerate_docker_file = not i.get('docker_noregenerate', False) + recreate_docker_image = i.get('docker_recreate', False) if i.get('docker_skip_build', False): - noregenerate_docker_file = True - norecreate_docker_image = True + regenerate_docker_file = False + recreate_docker_image = False env['MLC_DOCKER_SKIP_BUILD'] = 'yes' # Prune unnecessary Docker-related input keys r = prune_input({'input': i, 'extra_keys_starts_with': ['docker_']}) f_run_cmd = r['new_input'] + print(f"regenerate_docker_file = {regenerate_docker_file}") # Regenerate Dockerfile if required - if not noregenerate_docker_file: + if regenerate_docker_file: r = dockerfile(self_module, i) if r['return'] > 0: return r @@ -344,7 +346,7 @@ def docker_run(self_module, i): # Execute the Docker container mlc_docker_input = { 'action': 'run', 'automation': 'script', 'tags': 'run,docker,container', - 'recreate': 'yes' if not norecreate_docker_image else 'no', + 'recreate': recreate_docker_image, 'env': env, 'interactive': interactive, 'mounts': mounts, 'detached': detached, 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, 'v': verbose, 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index f5b4ce14b..ad045dbf2 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -76,6 +76,8 @@ def prepare_docker_inputs(input_params, docker_settings, script_path): 'dockerfiles', f"{docker_filename_suffix}.Dockerfile") + docker_inputs['file_path'] = dockerfile_path + return docker_inputs, dockerfile_path diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 491c86976..79467f73e 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -443,8 +443,8 @@ def preprocess(i): f.close() - # f = open(env['MLC_DOCKERFILE_WITH_PATH'], "r") - # print(f.read()) + #f = open(env['MLC_DOCKERFILE_WITH_PATH'], "r") + #print(f.read()) return {'return': 0} From 9a37a988561b2657a5f0e1f3864d88d8cae62332 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:07:28 +0530 Subject: [PATCH 05/14] Fix mlc docker run args --- automation/script/docker.py | 6 ++-- automation/script/docker_utils.py | 35 ++++++++++++++----- script/app-mlperf-inference/meta.yaml | 4 +-- .../meta.yaml | 2 +- .../meta.yaml | 2 +- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index fccac5eae..b70ee0781 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -192,8 +192,6 @@ def docker_run(self_module, i): # Extract and handle basic inputs quiet = i.get('quiet', False) - detached = i.get('docker_detached', i.get('docker_dt', 'no')) - interactive = i.get('docker_interactive', i.get('docker_it', '')) verbose = i.get('v', False) show_time = i.get('show_time', False) @@ -329,7 +327,7 @@ def docker_run(self_module, i): # Prepare Docker-specific inputs docker_inputs, dockerfile_path = prepare_docker_inputs( - i, docker_settings, script_path) + i, docker_settings, script_path, True) if docker_inputs is None: return {'return': 1, 'error': 'Error preparing Docker inputs'} @@ -347,7 +345,7 @@ def docker_run(self_module, i): mlc_docker_input = { 'action': 'run', 'automation': 'script', 'tags': 'run,docker,container', 'recreate': recreate_docker_image, - 'env': env, 'interactive': interactive, 'mounts': mounts, 'detached': detached, + 'env': env, 'mounts': mounts, 'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, 'v': verbose, 'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}}, **docker_inputs diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index ad045dbf2..46dcdd44d 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -37,7 +37,7 @@ def process_mounts(mounts, env, i, docker_settings): return None -def prepare_docker_inputs(input_params, docker_settings, script_path): +def prepare_docker_inputs(input_params, docker_settings, script_path, run_stage=False): """ Prepares Docker-specific inputs such as Dockerfile path and runtime options. @@ -49,19 +49,32 @@ def prepare_docker_inputs(input_params, docker_settings, script_path): Returns: Tuple with Docker inputs dictionary and Dockerfile path or None in case of an error. """ + + keys = [ + "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", + "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", + "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" + ] + + if run_stage: + keys += [ + "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token", + "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive", + "dt", "it" + ] # Collect Dockerfile inputs docker_inputs = { key: input_params.get( f"docker_{key}", docker_settings.get( key, get_docker_default(key))) - for key in [ - "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", - "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", - "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" - ] + for key in keys if (value := input_params.get(f"docker_{key}", docker_settings.get(key, get_docker_default(key)))) is not None } + if docker_inputs.get('detached', docker_inputs.get('dt')): + docker_inputs['interactive'] = False + docker_inputs['detached'] = True + # Determine Dockerfile suffix and path docker_base_image = docker_inputs.get('base_image') docker_path = docker_inputs.get('path') @@ -246,13 +259,19 @@ def rebuild_flags( def get_docker_default(key): defaults = { "mlc_repo": "mlcommons@mlperf-automations", - "mlc_repo_branch": "mlc", + "mlc_repo_branch": "dev", "os": "ubuntu", "os_version": "24.04", "fake_run_deps": False, "run_final_cmds": [], "skip_run_cmd": False, - "image_tag_extra": "-latest" + "image_tag_extra": "-latest", + "skip_run_cmd": False, + "pre_run_cmds": [], + "run_cmd_prefix": '', + "port_maps": [], + "detached": False, + "interactive": True } if key in defaults: return defaults[key] diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index b731f13f2..975bae164 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1856,8 +1856,8 @@ docker: MLC_MLPERF_INFERENCE_SUBMISSION_DIR: [ on ] pre_run_cmds: - #- cm pull repo && mlcr --tags=get,git,repo,_repo.https://github.com/GATEOverflow/inference_results_v4.0.git --update - - cm pull repo + #- mlc pull repo && mlcr --tags=get,git,repo,_repo.https://github.com/GATEOverflow/inference_results_v4.0.git --update + - mlc pull repo mounts: - "${{ MLC_DATASET_IMAGENET_PATH }}:${{ MLC_DATASET_IMAGENET_PATH }}" - "${{ MLC_DATASET_OPENIMAGES_PATH }}:${{ MLC_DATASET_OPENIMAGES_PATH }}" diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml index f74d99e9a..f3f723d72 100644 --- a/script/build-mlperf-inference-server-nvidia/meta.yaml +++ b/script/build-mlperf-inference-server-nvidia/meta.yaml @@ -383,7 +383,7 @@ docker: - yes pre_run_cmds: - - cm pull repo + - mlc pull repo run_cmd_prefix: sudo apt remove -y cmake mounts: - "${{ IMAGENET_PATH }}:/data/imagenet-val" diff --git a/script/generate-mlperf-inference-submission/meta.yaml b/script/generate-mlperf-inference-submission/meta.yaml index 20f7f3594..51f45bba7 100644 --- a/script/generate-mlperf-inference-submission/meta.yaml +++ b/script/generate-mlperf-inference-submission/meta.yaml @@ -56,7 +56,7 @@ docker: os: ubuntu os_version: '22.04' pre_run_cmds: - - cm pull repo + - mlc pull repo real_run: false use_host_group_id: true use_host_user_id: true From 64937dc4783175ff6bb023406849ef7b2168c743 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 24 Jan 2025 00:37:52 +0000 Subject: [PATCH 06/14] [Automated Commit] Format Codebase --- automation/script/docker.py | 2 +- automation/script/docker_utils.py | 23 ++++++++++++----------- script/build-dockerfile/customize.py | 4 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index b70ee0781..f77462fa9 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -196,7 +196,7 @@ def docker_run(self_module, i): show_time = i.get('show_time', False) env = i.get('env', {}) - + regenerate_docker_file = not i.get('docker_noregenerate', False) recreate_docker_image = i.get('docker_recreate', False) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 46dcdd44d..6df852c3d 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -37,7 +37,8 @@ def process_mounts(mounts, env, i, docker_settings): return None -def prepare_docker_inputs(input_params, docker_settings, script_path, run_stage=False): +def prepare_docker_inputs(input_params, docker_settings, + script_path, run_stage=False): """ Prepares Docker-specific inputs such as Dockerfile path and runtime options. @@ -51,17 +52,17 @@ def prepare_docker_inputs(input_params, docker_settings, script_path, run_stage= """ keys = [ - "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", - "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", - "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" - ] + "mlc_repo", "mlc_repo_branch", "base_image", "os", "os_version", + "mlc_repos", "skip_mlc_sys_upgrade", "extra_sys_deps", + "gh_token", "fake_run_deps", "run_final_cmds", "real_run", "copy_files", "path" + ] if run_stage: keys += [ - "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token", - "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive", - "dt", "it" - ] + "skip_run_cmd", "pre_run_cmds", "run_cmd_prefix", "all_gpus", "num_gpus", "device", "gh_token", + "port_maps", "shm_size", "pass_user_id", "pass_user_group", "extra_run_args", "detached", "interactive", + "dt", "it" + ] # Collect Dockerfile inputs docker_inputs = { key: input_params.get( @@ -267,10 +268,10 @@ def get_docker_default(key): "skip_run_cmd": False, "image_tag_extra": "-latest", "skip_run_cmd": False, - "pre_run_cmds": [], + "pre_run_cmds": [], "run_cmd_prefix": '', "port_maps": [], - "detached": False, + "detached": False, "interactive": True } if key in defaults: diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 79467f73e..491c86976 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -443,8 +443,8 @@ def preprocess(i): f.close() - #f = open(env['MLC_DOCKERFILE_WITH_PATH'], "r") - #print(f.read()) + # f = open(env['MLC_DOCKERFILE_WITH_PATH'], "r") + # print(f.read()) return {'return': 0} From 4d67a1788d8c90e7dcf2ac5e0eb0057aff0f7105 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:25:27 +0530 Subject: [PATCH 07/14] Added systemctl for ubuntu docker --- script/build-dockerfile/dockerinfo.json | 2 +- script/get-platform-details/run.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/build-dockerfile/dockerinfo.json b/script/build-dockerfile/dockerinfo.json index 4cc98e35c..05d712418 100644 --- a/script/build-dockerfile/dockerinfo.json +++ b/script/build-dockerfile/dockerinfo.json @@ -29,7 +29,7 @@ "package-manager-update-cmd": "apt-get update -y", "package-manager-get-cmd": "apt-get install -y", "packages": [ - "python3", "python3-pip", "git", "sudo", "wget", "python3-venv" + "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl" ], "versions": { "18.04": { diff --git a/script/get-platform-details/run.sh b/script/get-platform-details/run.sh index 8f6e30d49..2336146e1 100644 --- a/script/get-platform-details/run.sh +++ b/script/get-platform-details/run.sh @@ -64,12 +64,12 @@ echo "------------------------------------------------------------" >> $OUTPUT_F echo "11. Systemd service manager version" >> $OUTPUT_FILE eval "systemctl --version | head -n 1" >> $OUTPUT_FILE -test $? -eq 0 || exit $? +#test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "12. Services, from systemctl list-unit-files" >> $OUTPUT_FILE eval "systemctl list-unit-files" >> $OUTPUT_FILE -test $? -eq 0 || exit $? +#test $? -eq 0 || exit $? echo "------------------------------------------------------------" >> $OUTPUT_FILE echo "13. Linux kernel boot-time arguments, from /proc/cmdline" >> $OUTPUT_FILE From 0948faf5c1988b648f7b3f7c0918c777786ac9d6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:30:01 +0530 Subject: [PATCH 08/14] Fix formatting error --- automation/script/docker_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 6df852c3d..2b5b07b27 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -128,11 +128,9 @@ def update_docker_paths(path, mounts=None, force_target_path=''): # Determine the mount string based on whether the path is a file or # directory. if os.path.isfile(host_path) or not os.path.isdir(host_path): - mount_entry = f"{ - os.path.dirname(host_path)}: { - os.path.dirname(container_path)}" + mount_entry = f"""{os.path.dirname(host_path)}: {os.path.dirname(container_path)}""" else: - mount_entry = f"{host_path}:{container_path}" + mount_entry = f"""{host_path}:{container_path}""" # Add the mount entry to the mounts list if it's not already present. if mounts is not None: From cb9319fc1f6b87b8d45a415358a6096bd942acec Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:35:48 +0530 Subject: [PATCH 09/14] Fix automotive docker cmd for mlc --- script/app-mlperf-automotive/meta.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 80e62af84..780d34351 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -87,19 +87,20 @@ deps: - python - python3 - # Use cmind inside CM scripts - - tags: get,generic-python-lib,_package.cmind + # Use mlc inside scripts + - tags: get,generic-python-lib,_package.mlc - tags: get,mlperf,inference,utils docker: - mlc_repo: gateoverflow@cm4mlops + mlc_repo: mlcommons@mlperf-automations + mlc_repo_branch: dev use_host_group_id: True use_host_user_id: True real_run: false interactive: True - mlc_repos: 'cm pull repo mlcommons@cm4abtf --checkout=poc' + mlc_repos: 'mlc pull repo mlcommons@cm4abtf --checkout=poc' deps: - tags: get,abtf,scratch,space mounts: From 665b5f514ae052ab1ac9746f3ad7d9c0e2158ebe Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:38:45 +0530 Subject: [PATCH 10/14] Fix automotive docker cmd for mlc --- script/app-mlperf-automotive/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 780d34351..e598874e6 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -88,7 +88,7 @@ deps: - python3 # Use mlc inside scripts - - tags: get,generic-python-lib,_package.mlc + #- tags: get,generic-python-lib,_package.mlcflow - tags: get,mlperf,inference,utils @@ -100,7 +100,7 @@ docker: use_host_user_id: True real_run: false interactive: True - mlc_repos: 'mlc pull repo mlcommons@cm4abtf --checkout=poc' + mlc_repos: 'mlc pull repo mlcommons@cm4abtf --branch=poc' deps: - tags: get,abtf,scratch,space mounts: From 43b6653d3dad18ad3d7fee0cba6d791cc39812d6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 06:42:36 +0530 Subject: [PATCH 11/14] Fix automotive docker cmd for mlc --- script/app-mlperf-automotive/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index e598874e6..be0ce6333 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -94,7 +94,7 @@ deps: docker: - mlc_repo: mlcommons@mlperf-automations + mlc_repo: gateoverflow@mlperf-automations mlc_repo_branch: dev use_host_group_id: True use_host_user_id: True From dd996df11f440d6cfbd4f67b7945003abdc9e402 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 01:30:36 +0000 Subject: [PATCH 12/14] Update customize.py --- script/run-mlperf-inference-app/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index a966adbc5..a9e7fcb9c 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -3,7 +3,7 @@ import json import shutil import subprocess -# import mlc +import mlc import copy import mlperf_utils @@ -276,7 +276,7 @@ def preprocess(i): for k in docker_extra_input: ii[k] = docker_extra_input[k] - mlc = i['automation'].action_object + #mlc = i['automation'].action_object # print(ii) # return {'return': 1} From d9255b733a889d39ff2abcbeb7b754bf9881520c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 24 Jan 2025 01:30:49 +0000 Subject: [PATCH 13/14] [Automated Commit] Format Codebase --- script/run-mlperf-inference-app/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index a9e7fcb9c..4c5db8014 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -276,7 +276,7 @@ def preprocess(i): for k in docker_extra_input: ii[k] = docker_extra_input[k] - #mlc = i['automation'].action_object + # mlc = i['automation'].action_object # print(ii) # return {'return': 1} From 2b1ee5015a90f504fe7e77b9a18e4e29f403e32f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 24 Jan 2025 01:31:29 +0000 Subject: [PATCH 14/14] Update customize.py --- script/run-mlperf-inference-app/customize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 4c5db8014..51e68e493 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -46,6 +46,7 @@ def preprocess(i): # Clean MLPerf inference submission summary files x = env.get('MLPERF_INFERENCE_SUBMISSION_SUMMARY', '') + if x != '': for y in summary_ext: z = x + y