Support mlc experiment script

.github/workflows/test-mlc-script-features.yml

@@ -50,18 +50,15 @@ jobs:
         mlcr get,wkhtmltopdf --quiet
 
     - name: Test versions
-      continue-on-error: true
       if: runner.os == 'linux'
       run: |
         mlcr get,generic-python-lib,_package.scipy --version=1.9.3 --quiet
         test $? -eq 0 || exit $?
         mlcr get,generic-python-lib,_package.scipy --version=1.9.2 --quiet
         test $? -eq 0 || exit $?
-        mlc find cache --tags=get,generic-python-lib,_package.scipy,version-1.9.3
+        mlc find cache --tags=get,generic-python-lib,_package.scipy,version-1.9.2
         test $? -eq 0 || exit $?
-        # Need to add find cache here
-        # mlcr get,generic-python-lib,_package.scipy --version=1.9.3 --quiet --only_execute_from_cache=True
-        # test $? -eq 0 || exit 0
+
 
     - name: Test python install from src
       run: |
@@ -94,6 +91,34 @@ jobs:
       run: |
         mlcr run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet
 
+  test_experiment:
+    runs-on: ${{ matrix.os }}
+      strategy:
+        fail-fast: false
+        matrix:
+          python-version: ["3.12", "3.8"]
+          os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+
+      steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Configure git longpaths (Windows)
+        if: matrix.os == 'windows-latest'
+        run: |
+          git config --system core.longpaths true
+      - name: Pull MLOps repository
+        run: |
+          pip install mlcflow
+          mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+
+      - name: Test mlc experiment script
+        run: |
+          mlc experiment script --tags=detect,os --quiet --exp.repeat,=1,2,3
+          mlc experiment script --tags=detect,cpu --quiet --exp.explore=2:10:2
+
   test_mlperf_retinanet_cpp_venv:
     runs-on: ubuntu-latest
     strategy:

automation/script/experiment.py

@@ -0,0 +1,121 @@
+from collections import defaultdict
+import os
+from mlc import utils
+from utils import *
+import logging
+from pathlib import PureWindowsPath, PurePosixPath
+import copy
+
+
+def experiment_run(self_module, i):
+    """
+    Automates the exploration runs of MLC scripts.
+
+    Args:
+        self_module: Reference to the current module for internal calls.
+        i: Dictionary containing input parameters for the experiment execution.
+
+    Returns:
+        Dictionary with the result of the operation. Keys:
+        - 'return': 0 on success, >0 on error.
+        - 'error': Error message (if any).
+    """
+
+    # Extract and handle basic inputs
+    quiet = i.get('quiet', False)
+    show_time = i.get('show_time', False)
+    logger = self_module.logger
+    env = i.get('env', {})
+    prune_result = prune_input(
+        {'input': i, 'extra_keys_starts_with': ['exp.']})
+    if prune_result['return'] > 0:
+        return prune_result
+
+    run_input = prune_result['new_input']
+    if run_input.get('exp'):
+        del (run_input['exp'])
+
+    r = convert_input(i)
+    if r.get('exp'):
+        exp = r['exp']
+    else:
+        exp = {}
+
+    cur_dir = os.getcwd()
+    r = self_module.search(i.copy())
+    if r['return'] > 0:
+        return r
+
+    lst = r['list']
+    if not lst:
+        return {'return': 1, 'error': 'No scripts were found'}
+
+    # Process each artifact
+    for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')):
+        meta, script_path = artifact.meta, artifact.path
+        tags, script_alias, script_uid = meta.get(
+            "tags", []), meta.get(
+            'alias', ''), meta.get(
+            'uid', '')
+
+        # Execute the experiment script
+        mlc_script_input = {
+            'action': 'run', 'target': 'script'
+        }
+        if exp:
+            for key in exp:
+                ii = {**mlc_script_input, **run_input}
+                if isinstance(exp[key], list):
+                    for val in exp[key]:
+                        ii[key] = val
+                        r = self_module.action_object.access(ii)
+                        if r['return'] > 0:
+                            return r
+                elif isinstance(exp[key], dict):
+                    return {
+                        'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'}
+                else:
+                    ii[key] = exp[key]
+                    r = self_module.action_object.access(ii)
+                    if r['return'] > 0:
+                        return r
+
+    return {'return': 0}
+
+
+def parse_value(val):
+    if isinstance(val, list):
+        return [parse_value(v) for v in val]
+
+    val = str(val)
+
+    # Handle range inputs like 2:10 or 2:10:2
+    if ':' in val:
+        parts = val.split(':')
+        try:
+            parts = list(map(int, parts))
+            if len(parts) == 2:
+                return list(range(parts[0], parts[1] + 1))
+            elif len(parts) == 3:
+                return list(range(parts[0], parts[1] + 1, parts[2]))
+        except ValueError:
+            pass  # Not a valid range, fall through
+
+    # Convert to int if possible
+    if val.isdigit():
+        return int(val)
+
+    return val
+
+
+def convert_input(input_dict):
+    output = defaultdict(dict)
+
+    for key, value in input_dict.items():
+        if '.' in key:
+            main_key, sub_key = key.split('.', 1)
+            output[main_key][sub_key] = parse_value(value)
+        elif isinstance(value, dict):
+            output[key].update({k: parse_value(v) for k, v in value.items()})
+
+    return dict(output)

automation/script/module.py

@@ -4475,11 +4475,9 @@ def docker(self, i):
         return docker_run(self, i)
 
     ############################################################
-    # portion for experiment action.
-    # as of now, the experiment action directly calls the run action.
-    # in the future, we will add more functionality to the experiment action.
     def experiment(self, i):
-        return self.run(i)
+        from script.experiment import experiment_run
+        return experiment_run(self, i)
 
     ##########################################################################