From 48f74d5fdf17fd9eac1cf1663fb0ba72de71ba88 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 1 Apr 2025 22:24:15 +0530 Subject: [PATCH 01/14] fix for dmidecode --- script/get-mlperf-inference-sut-description/detect_memory.sh | 2 +- script/get-mlperf-inference-sut-description/meta.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/script/get-mlperf-inference-sut-description/detect_memory.sh b/script/get-mlperf-inference-sut-description/detect_memory.sh index 3d53fba78..317fd0283 100644 --- a/script/get-mlperf-inference-sut-description/detect_memory.sh +++ b/script/get-mlperf-inference-sut-description/detect_memory.sh @@ -3,4 +3,4 @@ if [[ ${MLC_SUDO_USER} == "yes" ]]; then ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE} fi -test $? -eq 0 || return $? +test $? -eq 0 || echo "Warning: Memory info is not recorded" diff --git a/script/get-mlperf-inference-sut-description/meta.yaml b/script/get-mlperf-inference-sut-description/meta.yaml index 5e935a0dc..573a6c842 100644 --- a/script/get-mlperf-inference-sut-description/meta.yaml +++ b/script/get-mlperf-inference-sut-description/meta.yaml @@ -29,6 +29,10 @@ deps: MLC_HOST_OS_TYPE: - linux tags: detect,sudo +- skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_dmidecode - env: MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH extra_cache_tags: mlperf,inference,sut,descriptions From 9af808373b80a278522d2fe00b1a72de418b8498 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 2 Apr 2025 17:38:38 +0530 Subject: [PATCH 02/14] support mounting file paths in docker --- automation/script/docker_utils.py | 15 +++++++--- .../meta.yaml | 29 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index a99f65324..92c2d62b0 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -78,10 +78,16 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): # Update container environment string and mappings if host_env_key: - container_env_string += f" --env.{host_env_key}={container_env_key} " + # We are modifying the input mapping variables wrt to the new container env variable. Is explicit passing of env variable necessary? + # container_env_string += f" --env.{host_env_key}={container_env_key} " for key, value in docker_input_mapping.items(): if value == host_env_key: - f_run_cmd[key] = container_env_key + # If we want to pass/mount a filepath, now its mounting its parent folder + # and in the input, along with the parent folder, filename is also passed to access + if os.path.basename(env[host_env_key]) != "": + f_run_cmd[key] = os.path.join(container_env_key, os.path.basename(env[host_env_key])) + else: + f_run_cmd[key] = container_env_key # Remove invalid mounts and construct mount string mounts = [item for item in mounts if item is not None] @@ -392,7 +398,7 @@ def get_docker_default(key): def get_host_path(value): # convert relative path to absolute path - value = convert_to_abs_path(value) + value = os.path.dirname(convert_to_abs_path(value)) path_split = value.split(os.sep) @@ -419,7 +425,8 @@ def get_container_path_script(i): def get_container_path(value, username="mlcuser"): # convert relative path to absolute path - value = convert_to_abs_path(value) + # if we are trying to mount a file, mount the parent folder. + value = os.path.dirname(convert_to_abs_path(value)) path_split = value.split(os.sep) diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index 6c5a8d417..a7a2f8b15 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -40,6 +40,34 @@ deps: MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION: - 'on' tags: preprocess,mlperf,inference,submission +docker: + mlc_repo: mlcommons@mlperf-automations + mlc_repo_branch: dev + extra_run_args: ' --cap-add SYS_ADMIN' + os: ubuntu + os_version: '22.04' + pre_run_cmds: + - mlc pull repo + real_run: false + use_host_group_id: true + use_host_user_id: true + deps: + - names: + - get-mlperf-submission-dir + skip_if_env: + MLC_MLPERF_INFERENCE_SUBMISSION_DIR: + - 'on' + tags: get,mlperf,submission,dir + - enable_if_env: + MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION: + - 'on' + tags: preprocess,mlperf,inference,submission + input_mapping: + submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR + submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE + mounts: + - ${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }} + - ${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}:${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }} input_mapping: extra_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS extra_checker_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS @@ -60,6 +88,7 @@ input_mapping: submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR submitter: MLC_MLPERF_SUBMITTER submitter_id: MLC_MLPERF_SUBMITTER_ID + submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE tar: MLC_TAR_SUBMISSION_DIR post_deps: - enable_if_env: From 6b737268ce863acc6f41fc99699d2f18f4be11df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Apr 2025 12:08:58 +0000 Subject: [PATCH 03/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 92c2d62b0..3fe3f96d9 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -83,9 +83,12 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): for key, value in docker_input_mapping.items(): if value == host_env_key: # If we want to pass/mount a filepath, now its mounting its parent folder - # and in the input, along with the parent folder, filename is also passed to access + # and in the input, along with the parent folder, filename + # is also passed to access if os.path.basename(env[host_env_key]) != "": - f_run_cmd[key] = os.path.join(container_env_key, os.path.basename(env[host_env_key])) + f_run_cmd[key] = os.path.join( + container_env_key, os.path.basename( + env[host_env_key])) else: f_run_cmd[key] = container_env_key From 845b141470e6486fbc369c5b8e3e0429d85c1f38 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 9 Apr 2025 02:36:13 +0530 Subject: [PATCH 04/14] update logic flow --- automation/script/docker_utils.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 3fe3f96d9..ef02a5268 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -78,19 +78,10 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): # Update container environment string and mappings if host_env_key: - # We are modifying the input mapping variables wrt to the new container env variable. Is explicit passing of env variable necessary? - # container_env_string += f" --env.{host_env_key}={container_env_key} " + container_env_string += f" --env.{host_env_key}={container_env_key} " for key, value in docker_input_mapping.items(): if value == host_env_key: - # If we want to pass/mount a filepath, now its mounting its parent folder - # and in the input, along with the parent folder, filename - # is also passed to access - if os.path.basename(env[host_env_key]) != "": - f_run_cmd[key] = os.path.join( - container_env_key, os.path.basename( - env[host_env_key])) - else: - f_run_cmd[key] = container_env_key + f_run_cmd[key] = container_env_key # Remove invalid mounts and construct mount string mounts = [item for item in mounts if item is not None] @@ -429,7 +420,6 @@ def get_container_path_script(i): def get_container_path(value, username="mlcuser"): # convert relative path to absolute path # if we are trying to mount a file, mount the parent folder. - value = os.path.dirname(convert_to_abs_path(value)) path_split = value.split(os.sep) @@ -444,4 +434,4 @@ def get_container_path(value, username="mlcuser"): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - return target_path, target_path + return os.path.dirname(target_path), target_path From 5e9f7e46b66076f0195eabfc2606ab7624e6543c Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 9 Apr 2025 09:03:56 +0530 Subject: [PATCH 05/14] handle relative path in get_container_value --- automation/script/docker_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index ef02a5268..c9394f8b6 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -218,8 +218,7 @@ def update_container_paths(path, mounts=None, force_target_path=''): if not path: return '', '' # Return empty paths if no path is provided. - # Normalize and resolve the absolute path. - host_path = os.path.abspath(path) + host_path = path container_path = host_path # Default to the same path for containers. # Handle Windows-specific path conversion for Docker. @@ -420,6 +419,7 @@ def get_container_path_script(i): def get_container_path(value, username="mlcuser"): # convert relative path to absolute path # if we are trying to mount a file, mount the parent folder. + value = convert_to_abs_path(value) path_split = value.split(os.sep) From 5fb35f3782e7279965e0227e99670b821606b6eb Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 10 Apr 2025 16:17:31 +0530 Subject: [PATCH 06/14] handle extraction of parent directory properly --- automation/script/docker_utils.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index c9394f8b6..ba99f9bdc 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -1,7 +1,7 @@ import os from mlc import utils from utils import * -from pathlib import PureWindowsPath, PurePosixPath +from pathlib import PureWindowsPath, PurePosixPath, Path from script.docker_utils import * import copy @@ -11,7 +11,16 @@ def convert_to_abs_path(path): path = os.path.abspath(path) return path - +# gets parent directory if the path is a file. Works even though the file is not present and is just a path +def get_directory(path_str): + path = Path(path_str).resolve() + + # If it has a file extension, assume it's a file and return parent dir + if path.suffix: + return str(path.parent) + else: + return str(path) + def process_mounts(mounts, env, docker_settings, f_run_cmd): """ Processes and updates the Docker mounts based on the provided inputs and environment variables. @@ -391,7 +400,8 @@ def get_docker_default(key): def get_host_path(value): # convert relative path to absolute path - value = os.path.dirname(convert_to_abs_path(value)) + # if file path is given, return the parent folder path + value = get_directory(value) path_split = value.split(os.sep) @@ -434,4 +444,5 @@ def get_container_path(value, username="mlcuser"): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - return os.path.dirname(target_path), target_path + # new container path is the parent folder of the target path if it is a file + return get_directory(target_path), target_path From 8a5111f62049fbaf199a35c15213c0613dad6dcc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Apr 2025 10:47:58 +0000 Subject: [PATCH 07/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker_utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index ba99f9bdc..7b53f8026 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -11,16 +11,20 @@ def convert_to_abs_path(path): path = os.path.abspath(path) return path -# gets parent directory if the path is a file. Works even though the file is not present and is just a path +# gets parent directory if the path is a file. Works even though the file +# is not present and is just a path + + def get_directory(path_str): path = Path(path_str).resolve() - + # If it has a file extension, assume it's a file and return parent dir if path.suffix: return str(path.parent) else: return str(path) - + + def process_mounts(mounts, env, docker_settings, f_run_cmd): """ Processes and updates the Docker mounts based on the provided inputs and environment variables. @@ -444,5 +448,6 @@ def get_container_path(value, username="mlcuser"): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - # new container path is the parent folder of the target path if it is a file + # new container path is the parent folder of the target path if it is a + # file return get_directory(target_path), target_path From 3b730f8df5d890937b49b7ac25e3e66ae23bfc37 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Thu, 10 Apr 2025 16:18:54 +0530 Subject: [PATCH 08/14] file updated --- automation/script/docker_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index ba99f9bdc..4740aa589 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -14,7 +14,6 @@ def convert_to_abs_path(path): # gets parent directory if the path is a file. Works even though the file is not present and is just a path def get_directory(path_str): path = Path(path_str).resolve() - # If it has a file extension, assume it's a file and return parent dir if path.suffix: return str(path.parent) From 259536953d7420bd9bb6f0e32281900006dc57a6 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 14 Apr 2025 14:19:38 +0530 Subject: [PATCH 09/14] handle directory extraction using file and folder path env keys --- automation/script/docker.py | 14 ++++++- automation/script/docker_utils.py | 42 ++++++++++++------- .../meta.yaml | 4 ++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 8e07417b6..32e689950 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -294,12 +294,22 @@ def docker_run(self_module, i): 'docker_mounts', [])) # do we need a copy here? variations = meta.get('variations', {}) + docker_settings = meta.get('docker', {}) state['docker'] = docker_settings + # take the folder path as well as file path env variables from meta + file_path_env_keys = docker_settings.get('file_path_env_keys', []) + folder_path_env_keys = docker_settings.get('folder_path_env_keys', []) + print(f"file_path_env_keys: {file_path_env_keys}") + print(f"folder_path_env_keys: {folder_path_env_keys}") + # from time import sleep + # sleep(5) run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags + 'script_variation_tags': variation_tags, + 'file_path_env_keys': file_path_env_keys, + 'folder_path_env_keys': folder_path_env_keys } # Update state and handle variations @@ -371,7 +381,7 @@ def docker_run(self_module, i): for key in docker_input_mapping if key in i}) # Handle environment variable-based mounts - res = process_mounts(mounts, env, docker_settings, f_run_cmd) + res = process_mounts(mounts, env, docker_settings, f_run_cmd, run_state) if res['return'] > 0: return res docker_inputs['mounts'] = res['mounts'] diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index f6448f7ad..8c7a41112 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -24,7 +24,7 @@ def get_directory(path_str): return str(path) -def process_mounts(mounts, env, docker_settings, f_run_cmd): +def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): """ Processes and updates the Docker mounts based on the provided inputs and environment variables. @@ -44,6 +44,8 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): container_env_string = "" for index in range(len(mounts)): + extract_parent_folder = False + mount = mounts[index] # Locate the last ':' to separate the mount into host and container @@ -66,7 +68,11 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): for placeholder in host_placeholders: if placeholder in env: host_env_key = placeholder - new_host_mount = get_host_path(env[placeholder]) + # if the env variable is in the file_path_env_keys, then we need to get the parent folder path(set extract_parent_folder to True) + if placeholder in run_state['file_path_env_keys']: + # set extract_parent_folder to True + extract_parent_folder = True + new_host_mount = get_host_path(env[placeholder], extract_parent_folder) else: # Skip mount if variable is missing mounts[index] = None break @@ -76,8 +82,12 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): if container_placeholders: for placeholder in container_placeholders: if placeholder in env: + # if the env variable is in the folder_path_env_keys, then we need to get the parent folder path(set extract_parent_folder to True) + if placeholder in run_state['folder_path_env_keys']: + # set extract_parent_folder to True + extract_parent_folder = True new_container_mount, container_env_key = get_container_path( - env[placeholder], docker_settings.get('user', 'mlcuser')) + env[placeholder], docker_settings.get('user', 'mlcuser'), extract_parent_folder) else: # Skip mount if variable is missing mounts[index] = None break @@ -246,12 +256,7 @@ def update_container_paths(path, mounts=None, force_target_path=''): container_path = '/mlc-mount' + \ container_path if not force_target_path else force_target_path - # Determine the mount string based on whether the path is a file or - # directory. - if os.path.isfile(host_path) or not os.path.isdir(host_path): - mount_entry = f"""{os.path.dirname(host_path)}: {os.path.dirname(container_path)}""" - else: - mount_entry = f"""{host_path}:{container_path}""" + mount_entry = f"""{host_path}:{container_path}""" # Add the mount entry to the mounts list if it's not already present. if mounts is not None: @@ -401,10 +406,13 @@ def get_docker_default(key): return None -def get_host_path(value): +def get_host_path(value, extract_parent_folder=False): # convert relative path to absolute path - # if file path is given, return the parent folder path - value = get_directory(value) + value = convert_to_abs_path(value) + + # if extract_parent_folder is True, then we need to get the parent folder path + if extract_parent_folder: + value = get_directory(value) path_split = value.split(os.sep) @@ -429,7 +437,7 @@ def get_container_path_script(i): return {'return': 0, 'value_mnt': value_mnt, 'value_env': value_env} -def get_container_path(value, username="mlcuser"): +def get_container_path(value, username="mlcuser", extract_parent_folder=False): # convert relative path to absolute path # if we are trying to mount a file, mount the parent folder. value = convert_to_abs_path(value) @@ -447,6 +455,8 @@ def get_container_path(value, username="mlcuser"): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - # new container path is the parent folder of the target path if it is a - # file - return get_directory(target_path), target_path + # new container path is the parent folder of the target path if extract_parent_folder is True + if extract_parent_folder: + return get_directory(target_path), target_path + else: + return target_path, target_path diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index a7a2f8b15..5d7b6720a 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -65,6 +65,10 @@ docker: input_mapping: submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE + file_path_env_keys: + - MLPERF_INFERENCE_SUBMISSION_TAR_FILE + folder_path_env_keys: + - MLC_MLPERF_INFERENCE_SUBMISSION_DIR mounts: - ${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }} - ${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}:${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }} From 799500ec119fb89df3e6574dc0e9974682904ce5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Apr 2025 08:50:06 +0000 Subject: [PATCH 10/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker.py | 7 ++++++- automation/script/docker_utils.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 32e689950..f0c548445 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -381,7 +381,12 @@ def docker_run(self_module, i): for key in docker_input_mapping if key in i}) # Handle environment variable-based mounts - res = process_mounts(mounts, env, docker_settings, f_run_cmd, run_state) + res = process_mounts( + mounts, + env, + docker_settings, + f_run_cmd, + run_state) if res['return'] > 0: return res docker_inputs['mounts'] = res['mounts'] diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 8c7a41112..a94e1a010 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -68,11 +68,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): for placeholder in host_placeholders: if placeholder in env: host_env_key = placeholder - # if the env variable is in the file_path_env_keys, then we need to get the parent folder path(set extract_parent_folder to True) + # if the env variable is in the file_path_env_keys, then we + # need to get the parent folder path(set + # extract_parent_folder to True) if placeholder in run_state['file_path_env_keys']: # set extract_parent_folder to True extract_parent_folder = True - new_host_mount = get_host_path(env[placeholder], extract_parent_folder) + new_host_mount = get_host_path( + env[placeholder], extract_parent_folder) else: # Skip mount if variable is missing mounts[index] = None break @@ -82,7 +85,9 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): if container_placeholders: for placeholder in container_placeholders: if placeholder in env: - # if the env variable is in the folder_path_env_keys, then we need to get the parent folder path(set extract_parent_folder to True) + # if the env variable is in the folder_path_env_keys, then + # we need to get the parent folder path(set + # extract_parent_folder to True) if placeholder in run_state['folder_path_env_keys']: # set extract_parent_folder to True extract_parent_folder = True @@ -410,7 +415,8 @@ def get_host_path(value, extract_parent_folder=False): # convert relative path to absolute path value = convert_to_abs_path(value) - # if extract_parent_folder is True, then we need to get the parent folder path + # if extract_parent_folder is True, then we need to get the parent folder + # path if extract_parent_folder: value = get_directory(value) @@ -455,7 +461,8 @@ def get_container_path(value, username="mlcuser", extract_parent_folder=False): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - # new container path is the parent folder of the target path if extract_parent_folder is True + # new container path is the parent folder of the target path if + # extract_parent_folder is True if extract_parent_folder: return get_directory(target_path), target_path else: From 0e83b5cef137c956729bbccb683477df0b07b90b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 15 Apr 2025 08:48:10 +0530 Subject: [PATCH 11/14] clean code --- automation/script/docker.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index f0c548445..5e9eba40b 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -300,10 +300,6 @@ def docker_run(self_module, i): # take the folder path as well as file path env variables from meta file_path_env_keys = docker_settings.get('file_path_env_keys', []) folder_path_env_keys = docker_settings.get('folder_path_env_keys', []) - print(f"file_path_env_keys: {file_path_env_keys}") - print(f"folder_path_env_keys: {folder_path_env_keys}") - # from time import sleep - # sleep(5) run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}", From 7e127ac4f177d36f8f61f098faf664b8542d482f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 15 Apr 2025 15:37:07 +0530 Subject: [PATCH 12/14] file_path_env_keys made part of general meta --- automation/script/docker.py | 8 +++++--- script/run-mlperf-inference-submission-checker/meta.yaml | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 5e9eba40b..62f23011f 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -295,11 +295,13 @@ def docker_run(self_module, i): [])) # do we need a copy here? variations = meta.get('variations', {}) + # take the folder path as well as file path env variables from meta + file_path_env_keys = meta.get('file_path_env_keys', []) + folder_path_env_keys = meta.get('folder_path_env_keys', []) + docker_settings = meta.get('docker', {}) state['docker'] = docker_settings - # take the folder path as well as file path env variables from meta - file_path_env_keys = docker_settings.get('file_path_env_keys', []) - folder_path_env_keys = docker_settings.get('folder_path_env_keys', []) + run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}", diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index 5d7b6720a..8d9667ae6 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -6,6 +6,10 @@ category: MLPerf benchmark support clean_files: [] default_env: MLC_MLPERF_SHORT_RUN: 'no' +file_path_env_keys: + - MLPERF_INFERENCE_SUBMISSION_TAR_FILE +folder_path_env_keys: + - MLC_MLPERF_INFERENCE_SUBMISSION_DIR default_version: master deps: - names: @@ -65,10 +69,6 @@ docker: input_mapping: submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE - file_path_env_keys: - - MLPERF_INFERENCE_SUBMISSION_TAR_FILE - folder_path_env_keys: - - MLC_MLPERF_INFERENCE_SUBMISSION_DIR mounts: - ${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }} - ${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}:${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }} From 094657f0e8c38571c31b894f3e0c26eaacb005e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Apr 2025 10:08:31 +0000 Subject: [PATCH 13/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 62f23011f..2d2f47340 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -301,7 +301,7 @@ def docker_run(self_module, i): docker_settings = meta.get('docker', {}) state['docker'] = docker_settings - + run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}", From a288fe7e49e03c683d6e4ce475f3030abbb38ba2 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 15 Apr 2025 15:39:24 +0530 Subject: [PATCH 14/14] run gh action --- automation/script/docker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 2d2f47340..c2f65ee5c 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -301,7 +301,6 @@ def docker_run(self_module, i): docker_settings = meta.get('docker', {}) state['docker'] = docker_settings - run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}",