diff --git a/automation/script/docker.py b/automation/script/docker.py index 8e07417b6..c2f65ee5c 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -294,12 +294,19 @@ def docker_run(self_module, i): 'docker_mounts', [])) # do we need a copy here? variations = meta.get('variations', {}) + + # take the folder path as well as file path env variables from meta + file_path_env_keys = meta.get('file_path_env_keys', []) + folder_path_env_keys = meta.get('folder_path_env_keys', []) + docker_settings = meta.get('docker', {}) state['docker'] = docker_settings run_state = { 'deps': [], 'fake_deps': [], 'parent': None, 'script_id': f"{script_alias},{script_uid}", - 'script_variation_tags': variation_tags + 'script_variation_tags': variation_tags, + 'file_path_env_keys': file_path_env_keys, + 'folder_path_env_keys': folder_path_env_keys } # Update state and handle variations @@ -371,7 +378,12 @@ def docker_run(self_module, i): for key in docker_input_mapping if key in i}) # Handle environment variable-based mounts - res = process_mounts(mounts, env, docker_settings, f_run_cmd) + res = process_mounts( + mounts, + env, + docker_settings, + f_run_cmd, + run_state) if res['return'] > 0: return res docker_inputs['mounts'] = res['mounts'] diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index a99f65324..a94e1a010 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -1,7 +1,7 @@ import os from mlc import utils from utils import * -from pathlib import PureWindowsPath, PurePosixPath +from pathlib import PureWindowsPath, PurePosixPath, Path from script.docker_utils import * import copy @@ -11,8 +11,20 @@ def convert_to_abs_path(path): path = os.path.abspath(path) return path +# gets parent directory if the path is a file. Works even though the file +# is not present and is just a path -def process_mounts(mounts, env, docker_settings, f_run_cmd): + +def get_directory(path_str): + path = Path(path_str).resolve() + # If it has a file extension, assume it's a file and return parent dir + if path.suffix: + return str(path.parent) + else: + return str(path) + + +def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): """ Processes and updates the Docker mounts based on the provided inputs and environment variables. @@ -32,6 +44,8 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): container_env_string = "" for index in range(len(mounts)): + extract_parent_folder = False + mount = mounts[index] # Locate the last ':' to separate the mount into host and container @@ -54,7 +68,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): for placeholder in host_placeholders: if placeholder in env: host_env_key = placeholder - new_host_mount = get_host_path(env[placeholder]) + # if the env variable is in the file_path_env_keys, then we + # need to get the parent folder path(set + # extract_parent_folder to True) + if placeholder in run_state['file_path_env_keys']: + # set extract_parent_folder to True + extract_parent_folder = True + new_host_mount = get_host_path( + env[placeholder], extract_parent_folder) else: # Skip mount if variable is missing mounts[index] = None break @@ -64,8 +85,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd): if container_placeholders: for placeholder in container_placeholders: if placeholder in env: + # if the env variable is in the folder_path_env_keys, then + # we need to get the parent folder path(set + # extract_parent_folder to True) + if placeholder in run_state['folder_path_env_keys']: + # set extract_parent_folder to True + extract_parent_folder = True new_container_mount, container_env_key = get_container_path( - env[placeholder], docker_settings.get('user', 'mlcuser')) + env[placeholder], docker_settings.get('user', 'mlcuser'), extract_parent_folder) else: # Skip mount if variable is missing mounts[index] = None break @@ -218,8 +245,7 @@ def update_container_paths(path, mounts=None, force_target_path=''): if not path: return '', '' # Return empty paths if no path is provided. - # Normalize and resolve the absolute path. - host_path = os.path.abspath(path) + host_path = path container_path = host_path # Default to the same path for containers. # Handle Windows-specific path conversion for Docker. @@ -235,12 +261,7 @@ def update_container_paths(path, mounts=None, force_target_path=''): container_path = '/mlc-mount' + \ container_path if not force_target_path else force_target_path - # Determine the mount string based on whether the path is a file or - # directory. - if os.path.isfile(host_path) or not os.path.isdir(host_path): - mount_entry = f"""{os.path.dirname(host_path)}: {os.path.dirname(container_path)}""" - else: - mount_entry = f"""{host_path}:{container_path}""" + mount_entry = f"""{host_path}:{container_path}""" # Add the mount entry to the mounts list if it's not already present. if mounts is not None: @@ -390,10 +411,15 @@ def get_docker_default(key): return None -def get_host_path(value): +def get_host_path(value, extract_parent_folder=False): # convert relative path to absolute path value = convert_to_abs_path(value) + # if extract_parent_folder is True, then we need to get the parent folder + # path + if extract_parent_folder: + value = get_directory(value) + path_split = value.split(os.sep) new_value = '' @@ -417,8 +443,9 @@ def get_container_path_script(i): return {'return': 0, 'value_mnt': value_mnt, 'value_env': value_env} -def get_container_path(value, username="mlcuser"): +def get_container_path(value, username="mlcuser", extract_parent_folder=False): # convert relative path to absolute path + # if we are trying to mount a file, mount the parent folder. value = convert_to_abs_path(value) path_split = value.split(os.sep) @@ -434,4 +461,9 @@ def get_container_path(value, username="mlcuser"): return "/".join(new_path_split1), "/".join(new_path_split2) else: orig_path, target_path = update_container_paths(path=value) - return target_path, target_path + # new container path is the parent folder of the target path if + # extract_parent_folder is True + if extract_parent_folder: + return get_directory(target_path), target_path + else: + return target_path, target_path diff --git a/script/get-mlperf-inference-sut-description/detect_memory.sh b/script/get-mlperf-inference-sut-description/detect_memory.sh index 3d53fba78..317fd0283 100644 --- a/script/get-mlperf-inference-sut-description/detect_memory.sh +++ b/script/get-mlperf-inference-sut-description/detect_memory.sh @@ -3,4 +3,4 @@ if [[ ${MLC_SUDO_USER} == "yes" ]]; then ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE} fi -test $? -eq 0 || return $? +test $? -eq 0 || echo "Warning: Memory info is not recorded" diff --git a/script/get-mlperf-inference-sut-description/meta.yaml b/script/get-mlperf-inference-sut-description/meta.yaml index 5e935a0dc..573a6c842 100644 --- a/script/get-mlperf-inference-sut-description/meta.yaml +++ b/script/get-mlperf-inference-sut-description/meta.yaml @@ -29,6 +29,10 @@ deps: MLC_HOST_OS_TYPE: - linux tags: detect,sudo +- skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_dmidecode - env: MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH extra_cache_tags: mlperf,inference,sut,descriptions diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index 6c5a8d417..8d9667ae6 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -6,6 +6,10 @@ category: MLPerf benchmark support clean_files: [] default_env: MLC_MLPERF_SHORT_RUN: 'no' +file_path_env_keys: + - MLPERF_INFERENCE_SUBMISSION_TAR_FILE +folder_path_env_keys: + - MLC_MLPERF_INFERENCE_SUBMISSION_DIR default_version: master deps: - names: @@ -40,6 +44,34 @@ deps: MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION: - 'on' tags: preprocess,mlperf,inference,submission +docker: + mlc_repo: mlcommons@mlperf-automations + mlc_repo_branch: dev + extra_run_args: ' --cap-add SYS_ADMIN' + os: ubuntu + os_version: '22.04' + pre_run_cmds: + - mlc pull repo + real_run: false + use_host_group_id: true + use_host_user_id: true + deps: + - names: + - get-mlperf-submission-dir + skip_if_env: + MLC_MLPERF_INFERENCE_SUBMISSION_DIR: + - 'on' + tags: get,mlperf,submission,dir + - enable_if_env: + MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION: + - 'on' + tags: preprocess,mlperf,inference,submission + input_mapping: + submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR + submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE + mounts: + - ${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }} + - ${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}:${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }} input_mapping: extra_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS extra_checker_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS @@ -60,6 +92,7 @@ input_mapping: submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR submitter: MLC_MLPERF_SUBMITTER submitter_id: MLC_MLPERF_SUBMITTER_ID + submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE tar: MLC_TAR_SUBMISSION_DIR post_deps: - enable_if_env: