Fix for dmidecode + support mounting filepaths in docker

automation/script/docker.py

@@ -294,12 +294,19 @@ def docker_run(self_module, i):
                 'docker_mounts',
                 []))  # do we need a copy here?
         variations = meta.get('variations', {})
+
+        # take the folder path as well as file path env variables from meta
+        file_path_env_keys = meta.get('file_path_env_keys', [])
+        folder_path_env_keys = meta.get('folder_path_env_keys', [])
+
         docker_settings = meta.get('docker', {})
         state['docker'] = docker_settings
         run_state = {
             'deps': [], 'fake_deps': [], 'parent': None,
             'script_id': f"{script_alias},{script_uid}",
-            'script_variation_tags': variation_tags
+            'script_variation_tags': variation_tags,
+            'file_path_env_keys': file_path_env_keys,
+            'folder_path_env_keys': folder_path_env_keys
         }
 
         # Update state and handle variations
@@ -371,7 +378,12 @@ def docker_run(self_module, i):
                        for key in docker_input_mapping if key in i})
 
         # Handle environment variable-based mounts
-        res = process_mounts(mounts, env, docker_settings, f_run_cmd)
+        res = process_mounts(
+            mounts,
+            env,
+            docker_settings,
+            f_run_cmd,
+            run_state)
         if res['return'] > 0:
             return res
         docker_inputs['mounts'] = res['mounts']

automation/script/docker_utils.py

@@ -1,7 +1,7 @@
 import os
 from mlc import utils
 from utils import *
-from pathlib import PureWindowsPath, PurePosixPath
+from pathlib import PureWindowsPath, PurePosixPath, Path
 from script.docker_utils import *
 import copy
 
@@ -11,8 +11,20 @@ def convert_to_abs_path(path):
         path = os.path.abspath(path)
     return path
 
+# gets parent directory if the path is a file. Works even though the file
+# is not present and is just a path
 
-def process_mounts(mounts, env, docker_settings, f_run_cmd):
+
+def get_directory(path_str):
+    path = Path(path_str).resolve()
+    # If it has a file extension, assume it's a file and return parent dir
+    if path.suffix:
+        return str(path.parent)
+    else:
+        return str(path)
+
+
+def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state):
     """
     Processes and updates the Docker mounts based on the provided inputs and environment variables.
 
@@ -32,6 +44,8 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
     container_env_string = ""
 
     for index in range(len(mounts)):
+        extract_parent_folder = False
+
         mount = mounts[index]
 
         # Locate the last ':' to separate the mount into host and container
@@ -54,7 +68,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
             for placeholder in host_placeholders:
                 if placeholder in env:
                     host_env_key = placeholder
-                    new_host_mount = get_host_path(env[placeholder])
+                    # if the env variable is in the file_path_env_keys, then we
+                    # need to get the parent folder path(set
+                    # extract_parent_folder to True)
+                    if placeholder in run_state['file_path_env_keys']:
+                        # set extract_parent_folder to True
+                        extract_parent_folder = True
+                    new_host_mount = get_host_path(
+                        env[placeholder], extract_parent_folder)
                 else:  # Skip mount if variable is missing
                     mounts[index] = None
                     break
@@ -64,8 +85,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
         if container_placeholders:
             for placeholder in container_placeholders:
                 if placeholder in env:
+                    # if the env variable is in the folder_path_env_keys, then
+                    # we need to get the parent folder path(set
+                    # extract_parent_folder to True)
+                    if placeholder in run_state['folder_path_env_keys']:
+                        # set extract_parent_folder to True
+                        extract_parent_folder = True
                     new_container_mount, container_env_key = get_container_path(
-                        env[placeholder], docker_settings.get('user', 'mlcuser'))
+                        env[placeholder], docker_settings.get('user', 'mlcuser'), extract_parent_folder)
                 else:  # Skip mount if variable is missing
                     mounts[index] = None
                     break
@@ -218,8 +245,7 @@ def update_container_paths(path, mounts=None, force_target_path=''):
     if not path:
         return '', ''  # Return empty paths if no path is provided.
 
-    # Normalize and resolve the absolute path.
-    host_path = os.path.abspath(path)
+    host_path = path
     container_path = host_path  # Default to the same path for containers.
 
     # Handle Windows-specific path conversion for Docker.
@@ -235,12 +261,7 @@ def update_container_paths(path, mounts=None, force_target_path=''):
     container_path = '/mlc-mount' + \
         container_path if not force_target_path else force_target_path
 
-    # Determine the mount string based on whether the path is a file or
-    # directory.
-    if os.path.isfile(host_path) or not os.path.isdir(host_path):
-        mount_entry = f"""{os.path.dirname(host_path)}: {os.path.dirname(container_path)}"""
-    else:
-        mount_entry = f"""{host_path}:{container_path}"""
+    mount_entry = f"""{host_path}:{container_path}"""
 
     # Add the mount entry to the mounts list if it's not already present.
     if mounts is not None:
@@ -390,10 +411,15 @@ def get_docker_default(key):
     return None
 
 
-def get_host_path(value):
+def get_host_path(value, extract_parent_folder=False):
     # convert relative path to absolute path
     value = convert_to_abs_path(value)
 
+    # if extract_parent_folder is True, then we need to get the parent folder
+    # path
+    if extract_parent_folder:
+        value = get_directory(value)
+
     path_split = value.split(os.sep)
 
     new_value = ''
@@ -417,8 +443,9 @@ def get_container_path_script(i):
     return {'return': 0, 'value_mnt': value_mnt, 'value_env': value_env}
 
 
-def get_container_path(value, username="mlcuser"):
+def get_container_path(value, username="mlcuser", extract_parent_folder=False):
     # convert relative path to absolute path
+    # if we are trying to mount a file, mount the parent folder.
     value = convert_to_abs_path(value)
 
     path_split = value.split(os.sep)
@@ -434,4 +461,9 @@ def get_container_path(value, username="mlcuser"):
             return "/".join(new_path_split1), "/".join(new_path_split2)
     else:
         orig_path, target_path = update_container_paths(path=value)
-        return target_path, target_path
+        # new container path is the parent folder of the target path if
+        # extract_parent_folder is True
+        if extract_parent_folder:
+            return get_directory(target_path), target_path
+        else:
+            return target_path, target_path

script/get-mlperf-inference-sut-description/detect_memory.sh

@@ -3,4 +3,4 @@
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
   ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE}
 fi
-test $? -eq 0 || return $?
+test $? -eq 0 || echo "Warning: Memory info is not recorded"

script/get-mlperf-inference-sut-description/meta.yaml

@@ -29,6 +29,10 @@ deps:
     MLC_HOST_OS_TYPE:
     - linux
   tags: detect,sudo
+- skip_if_env:
+    MLC_SUDO_USER:
+    - 'no'
+  tags: get,sys-util,generic,_dmidecode
 - env:
     MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH
   extra_cache_tags: mlperf,inference,sut,descriptions

script/run-mlperf-inference-submission-checker/meta.yaml

@@ -6,6 +6,10 @@ category: MLPerf benchmark support
 clean_files: []
 default_env:
   MLC_MLPERF_SHORT_RUN: 'no'
+file_path_env_keys:
+  - MLPERF_INFERENCE_SUBMISSION_TAR_FILE
+folder_path_env_keys:
+  - MLC_MLPERF_INFERENCE_SUBMISSION_DIR
 default_version: master
 deps:
 - names:
@@ -40,6 +44,34 @@ deps:
     MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION:
     - 'on'
   tags: preprocess,mlperf,inference,submission
+docker:
+  mlc_repo: mlcommons@mlperf-automations
+  mlc_repo_branch: dev
+  extra_run_args: ' --cap-add SYS_ADMIN'
+  os: ubuntu
+  os_version: '22.04'
+  pre_run_cmds:
+    - mlc pull repo
+  real_run: false
+  use_host_group_id: true
+  use_host_user_id: true
+  deps:
+    - names:
+      - get-mlperf-submission-dir
+      skip_if_env:
+        MLC_MLPERF_INFERENCE_SUBMISSION_DIR:
+        - 'on'
+      tags: get,mlperf,submission,dir
+    - enable_if_env:
+        MLC_TMP_MLPERF_INFERENCE_PREPROCESS_SUBMISSION:
+        - 'on'
+      tags: preprocess,mlperf,inference,submission
+  input_mapping:
+    submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR
+    submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE
+  mounts:
+  - ${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ MLC_MLPERF_INFERENCE_SUBMISSION_DIR }}
+  - ${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}:${{ MLPERF_INFERENCE_SUBMISSION_TAR_FILE }}
 input_mapping:
   extra_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS
   extra_checker_args: MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS
@@ -60,6 +92,7 @@ input_mapping:
   submission_dir: MLC_MLPERF_INFERENCE_SUBMISSION_DIR
   submitter: MLC_MLPERF_SUBMITTER
   submitter_id: MLC_MLPERF_SUBMITTER_ID
+  submission_tar_file: MLPERF_INFERENCE_SUBMISSION_TAR_FILE
   tar: MLC_TAR_SUBMISSION_DIR
 post_deps:
 - enable_if_env: