Sync Dev

.github/workflows/build_wheels.yml

@@ -6,7 +6,6 @@ on:
   workflow_dispatch: {}
 
 jobs:
-
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/test-mlperf-inference-abtf-poc.yml

@@ -15,14 +15,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, ubuntu-24.04, macos-latest, macos-13, windows-latest]
+        os: [ubuntu-22.04, ubuntu-latest, macos-latest, macos-13, windows-latest]
         python-version: [ "3.8", "3.12" ]
         backend: [ "pytorch" ]
         implementation: [ "python" ]
         docker: [ "", " --docker --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --docker_dt" ]
         extra-args: [ "--adr.compiler.tags=gcc", "--env.MLC_MLPERF_LOADGEN_BUILD_FROM_SRC=off" ]
         exclude:
-        - os: ubuntu-24.04
+        - os: ubuntu-latest
           python-version: "3.8"
         - os: windows-latest
           python-version: "3.8"

automation/script/docker.py

@@ -20,7 +20,6 @@ def dockerfile(self_module, input_params):
     run_command_arc = prune_result['new_input']
     current_directory = os.getcwd()
     is_quiet_mode = input_params.get('quiet', False)
-    verbose = input_params.get('v', False)
     is_console_output = input_params.get('out') == 'con'
 
     # Step 2: Search for scripts
@@ -88,8 +87,7 @@ def dockerfile(self_module, input_params):
             new_state_keys_from_meta=[],  # Add state keys from meta if needed
             add_deps_recursive=add_deps_recursive,
             run_state=run_state,
-            recursion_spaces='',
-            verbose=verbose  # Set to True or False as needed
+            recursion_spaces=''
         )
         if update_variations_result['return'] > 0:
             return update_variations_result
@@ -109,7 +107,7 @@ def dockerfile(self_module, input_params):
         deps = docker_settings.get('build_deps', [])
         if deps:
             r = self_module._run_deps(
-                deps, [], env, {}, {}, {}, {}, '', [], '', False, '', verbose,
+                deps, [], env, {}, {}, {}, {}, '', [], '', False, '',
                 show_time, ' ', run_state)
             if r['return'] > 0:
                 return r
@@ -180,8 +178,34 @@ def dockerfile(self_module, input_params):
             'comments': comments, 'run_cmd': f"{run_command_string} --quiet",
             'script_tags': input_params.get('tags'), 'env': env,
             'dockerfile_env': dockerfile_env,
-            'quiet': True, 'v': input_params.get('v', False), 'real_run': True
+            'quiet': True, 'real_run': True
         }
+
+        docker_v = False
+        docker_s = False
+        if is_true(input_params.get(
+                'docker_v', input_params.get('docker_verbose', False))):
+            docker_v = True
+        if is_true(input_params.get(
+                'docker_s', input_params.get('docker_silent', False))):
+            docker_s = True
+
+        if docker_s and docker_v:
+            logger.warning(
+                "Both verbose and silent is set to True. Verbose will take precedence.")
+            docker_s = False
+
+        if not docker_s and not docker_v:
+            if logger.level == logging.DEBUG:
+                docker_v = True
+            elif logger.level == logging.WARNING:
+                docker_s = True
+
+        if docker_s:
+            mlc_docker_input['run_cmd'] += ' -s'
+        elif docker_v:
+            mlc_docker_input['run_cmd'] += ' -v'
+
         mlc_docker_input.update(docker_inputs)
 
         dockerfile_result = self_module.action_object.access(mlc_docker_input)
@@ -209,7 +233,6 @@ def docker_run(self_module, i):
 
     # Extract and handle basic inputs
     quiet = i.get('quiet', False)
-    verbose = i.get('v', False)
     show_time = i.get('show_time', False)
     logger = self_module.logger
     env = i.get('env', {})
@@ -271,12 +294,19 @@ def docker_run(self_module, i):
                 'docker_mounts',
                 []))  # do we need a copy here?
         variations = meta.get('variations', {})
+
+        # take the folder path as well as file path env variables from meta
+        file_path_env_keys = meta.get('file_path_env_keys', [])
+        folder_path_env_keys = meta.get('folder_path_env_keys', [])
+
         docker_settings = meta.get('docker', {})
         state['docker'] = docker_settings
         run_state = {
             'deps': [], 'fake_deps': [], 'parent': None,
             'script_id': f"{script_alias},{script_uid}",
-            'script_variation_tags': variation_tags
+            'script_variation_tags': variation_tags,
+            'file_path_env_keys': file_path_env_keys,
+            'folder_path_env_keys': folder_path_env_keys
         }
 
         # Update state and handle variations
@@ -296,8 +326,7 @@ def docker_run(self_module, i):
             posthook_deps=[],
             new_env_keys_from_meta=[],
             new_state_keys_from_meta=[],
-            add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='',
-            verbose=False)
+            add_deps_recursive=add_deps_recursive, run_state=run_state, recursion_spaces='')
         if r['return'] > 0:
             return r
 
@@ -306,7 +335,7 @@ def docker_run(self_module, i):
         deps = docker_settings.get('deps', [])
         if deps:
             r = self_module._run_deps(
-                deps, [], env, {}, {}, {}, {}, '', [], '', False, '', verbose,
+                deps, [], env, {}, {}, {}, {}, '', [], '', False, '',
                 show_time, ' ', run_state)
             if r['return'] > 0:
                 return r
@@ -349,7 +378,12 @@ def docker_run(self_module, i):
                        for key in docker_input_mapping if key in i})
 
         # Handle environment variable-based mounts
-        res = process_mounts(mounts, env, docker_settings, f_run_cmd)
+        res = process_mounts(
+            mounts,
+            env,
+            docker_settings,
+            f_run_cmd,
+            run_state)
         if res['return'] > 0:
             return res
         docker_inputs['mounts'] = res['mounts']
@@ -374,7 +408,7 @@ def docker_run(self_module, i):
             'action': 'run', 'target': 'script', 'tags': 'run,docker,container',
             'rebuild': rebuild_docker_image,
             'env': env, 'mounts': mounts,
-            'script_tags': i.get('tags'), 'run_cmd': final_run_cmd, 'v': verbose,
+            'script_tags': i.get('tags'), 'run_cmd': final_run_cmd,
             'quiet': True, 'real_run': True, 'add_deps_recursive': {'build-docker-image': {'dockerfile': dockerfile_path}},
             **docker_inputs
         }

automation/script/docker_utils.py

@@ -1,12 +1,30 @@
 import os
 from mlc import utils
 from utils import *
-from pathlib import PureWindowsPath, PurePosixPath
+from pathlib import PureWindowsPath, PurePosixPath, Path
 from script.docker_utils import *
 import copy
 
 
-def process_mounts(mounts, env, docker_settings, f_run_cmd):
+def convert_to_abs_path(path):
+    if not os.path.isabs(path):
+        path = os.path.abspath(path)
+    return path
+
+# gets parent directory if the path is a file. Works even though the file
+# is not present and is just a path
+
+
+def get_directory(path_str):
+    path = Path(path_str).resolve()
+    # If it has a file extension, assume it's a file and return parent dir
+    if path.suffix:
+        return str(path.parent)
+    else:
+        return str(path)
+
+
+def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state):
     """
     Processes and updates the Docker mounts based on the provided inputs and environment variables.
 
@@ -26,6 +44,8 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
     container_env_string = ""
 
     for index in range(len(mounts)):
+        extract_parent_folder = False
+
         mount = mounts[index]
 
         # Locate the last ':' to separate the mount into host and container
@@ -48,7 +68,14 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
             for placeholder in host_placeholders:
                 if placeholder in env:
                     host_env_key = placeholder
-                    new_host_mount = get_host_path(env[placeholder])
+                    # if the env variable is in the file_path_env_keys, then we
+                    # need to get the parent folder path(set
+                    # extract_parent_folder to True)
+                    if placeholder in run_state['file_path_env_keys']:
+                        # set extract_parent_folder to True
+                        extract_parent_folder = True
+                    new_host_mount = get_host_path(
+                        env[placeholder], extract_parent_folder)
                 else:  # Skip mount if variable is missing
                     mounts[index] = None
                     break
@@ -58,12 +85,17 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd):
         if container_placeholders:
             for placeholder in container_placeholders:
                 if placeholder in env:
+                    # if the env variable is in the folder_path_env_keys, then
+                    # we need to get the parent folder path(set
+                    # extract_parent_folder to True)
+                    if placeholder in run_state['folder_path_env_keys']:
+                        # set extract_parent_folder to True
+                        extract_parent_folder = True
                     new_container_mount, container_env_key = get_container_path(
-                        env[placeholder], docker_settings.get('user', 'mlcuser'))
+                        env[placeholder], docker_settings.get('user', 'mlcuser'), extract_parent_folder)
                 else:  # Skip mount if variable is missing
                     mounts[index] = None
                     break
-
         # Skip further processing if the mount was invalid
         if mounts[index] is None:
             continue
@@ -213,8 +245,7 @@ def update_container_paths(path, mounts=None, force_target_path=''):
     if not path:
         return '', ''  # Return empty paths if no path is provided.
 
-    # Normalize and resolve the absolute path.
-    host_path = os.path.abspath(path)
+    host_path = path
     container_path = host_path  # Default to the same path for containers.
 
     # Handle Windows-specific path conversion for Docker.
@@ -230,18 +261,12 @@ def update_container_paths(path, mounts=None, force_target_path=''):
     container_path = '/mlc-mount' + \
         container_path if not force_target_path else force_target_path
 
-    # Determine the mount string based on whether the path is a file or
-    # directory.
-    if os.path.isfile(host_path) or not os.path.isdir(host_path):
-        mount_entry = f"""{os.path.dirname(host_path)}: {os.path.dirname(container_path)}"""
-    else:
-        mount_entry = f"""{host_path}:{container_path}"""
+    mount_entry = f"""{host_path}:{container_path}"""
 
     # Add the mount entry to the mounts list if it's not already present.
     if mounts is not None:
         if all(mount.lower() != mount_entry.lower() for mount in mounts):
             mounts.append(mount_entry)
-
     return host_path, container_path
 
 
@@ -344,7 +369,10 @@ def rebuild_flags(
                 command_line += f" --{full_key},={list_values}"
             # Process scalar values.
             else:
-                command_line += f" --{full_key}={quote}{str(value)}{quote}"
+                if full_key in ['s', 'v']:
+                    command_line += f" -{full_key}"
+                else:
+                    command_line += f" --{full_key}={quote}{str(value)}{quote}"
 
         return command_line
 
@@ -383,10 +411,16 @@ def get_docker_default(key):
     return None
 
 
-def get_host_path(value):
+def get_host_path(value, extract_parent_folder=False):
+    # convert relative path to absolute path
+    value = convert_to_abs_path(value)
+
+    # if extract_parent_folder is True, then we need to get the parent folder
+    # path
+    if extract_parent_folder:
+        value = get_directory(value)
+
     path_split = value.split(os.sep)
-    if len(path_split) == 1:
-        return value
 
     new_value = ''
     if "cache" in path_split and "local":
@@ -409,10 +443,12 @@ def get_container_path_script(i):
     return {'return': 0, 'value_mnt': value_mnt, 'value_env': value_env}
 
 
-def get_container_path(value, username="mlcuser"):
+def get_container_path(value, username="mlcuser", extract_parent_folder=False):
+    # convert relative path to absolute path
+    # if we are trying to mount a file, mount the parent folder.
+    value = convert_to_abs_path(value)
+
     path_split = value.split(os.sep)
-    if len(path_split) == 1:
-        return value, value
 
     new_value = ''
     if "cache" in path_split and "local" in path_split:
@@ -425,4 +461,9 @@ def get_container_path(value, username="mlcuser"):
             return "/".join(new_path_split1), "/".join(new_path_split2)
     else:
         orig_path, target_path = update_container_paths(path=value)
-        return target_path, target_path
+        # new container path is the parent folder of the target path if
+        # extract_parent_folder is True
+        if extract_parent_folder:
+            return get_directory(target_path), target_path
+        else:
+            return target_path, target_path