From d57683d4c4caf8daab31dd159f6dc0553fe24419 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 17 May 2025 05:33:33 +0530 Subject: [PATCH 1/3] Support mlc experiment script --- automation/script/experiment.py | 122 ++++++++++++++++++++++++++++++++ automation/script/module.py | 6 +- 2 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 automation/script/experiment.py diff --git a/automation/script/experiment.py b/automation/script/experiment.py new file mode 100644 index 000000000..63ac369d8 --- /dev/null +++ b/automation/script/experiment.py @@ -0,0 +1,122 @@ +import os +from mlc import utils +from utils import * +import logging +from pathlib import PureWindowsPath, PurePosixPath +import copy + + +def experiment_run(self_module, i): + """ + Automates the exploration runs of MLC scripts. + + Args: + self_module: Reference to the current module for internal calls. + i: Dictionary containing input parameters for the experiment execution. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = i.get('quiet', False) + show_time = i.get('show_time', False) + logger = self_module.logger + env = i.get('env', {}) + prune_result = prune_input( + {'input': i, 'extra_keys_starts_with': ['exp.']}) + if prune_result['return'] > 0: + return prune_result + + run_input = prune_result['new_input'] + if run_input.get('exp'): + del(run_input['exp']) + + r = convert_input(i) + if r.get('exp'): + exp = r['exp'] + else: + exp = {} + + cur_dir = os.getcwd() + r = self_module.search(i.copy()) + if r['return'] > 0: + return r + + lst = r['list'] + if not lst: + return {'return': 1, 'error': 'No scripts were found'} + + + # Process each artifact + for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): + meta, script_path = artifact.meta, artifact.path + tags, script_alias, script_uid = meta.get( + "tags", []), meta.get( + 'alias', ''), meta.get( + 'uid', '') + + + # Execute the experiment script + mlc_script_input = { + 'action': 'run', 'target': 'script' + } + if exp: + for key in exp: + ii = {**mlc_script_input, **run_input} + if isinstance(exp[key], list): + for val in exp[key]: + ii[key] = val + r = self_module.action_object.access(ii) + if r['return'] > 0: + return r + elif isinstance(exp[key], dict): + return {'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} + else: + ii[key] = exp[key] + r = self_module.action_object.access(ii) + if r['return'] > 0: + return r + + return {'return': 0} + +from collections import defaultdict + +def parse_value(val): + if isinstance(val, list): + return [parse_value(v) for v in val] + + val = str(val) + + # Handle range inputs like 2:10 or 2:10:2 + if ':' in val: + parts = val.split(':') + try: + parts = list(map(int, parts)) + if len(parts) == 2: + return list(range(parts[0], parts[1] + 1)) + elif len(parts) == 3: + return list(range(parts[0], parts[1] + 1, parts[2])) + except ValueError: + pass # Not a valid range, fall through + + # Convert to int if possible + if val.isdigit(): + return int(val) + + return val + +def convert_input(input_dict): + output = defaultdict(dict) + + for key, value in input_dict.items(): + if '.' in key: + main_key, sub_key = key.split('.', 1) + output[main_key][sub_key] = parse_value(value) + elif isinstance(value, dict): + output[key].update({k: parse_value(v) for k, v in value.items()}) + + return dict(output) + diff --git a/automation/script/module.py b/automation/script/module.py index 5cac406ae..dcd709271 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4475,11 +4475,9 @@ def docker(self, i): return docker_run(self, i) ############################################################ - # portion for experiment action. - # as of now, the experiment action directly calls the run action. - # in the future, we will add more functionality to the experiment action. def experiment(self, i): - return self.run(i) + from script.experiment import experiment_run + return experiment_run(self, i) ########################################################################## From b9d8ee270b219e97cd7a5c2d91b548aa6a556cc6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 17 May 2025 00:03:52 +0000 Subject: [PATCH 2/3] [Automated Commit] Format Codebase [skip ci] --- automation/script/experiment.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index 63ac369d8..5195ce056 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -1,3 +1,4 @@ +from collections import defaultdict import os from mlc import utils from utils import * @@ -32,14 +33,14 @@ def experiment_run(self_module, i): run_input = prune_result['new_input'] if run_input.get('exp'): - del(run_input['exp']) - + del (run_input['exp']) + r = convert_input(i) if r.get('exp'): exp = r['exp'] else: exp = {} - + cur_dir = os.getcwd() r = self_module.search(i.copy()) if r['return'] > 0: @@ -49,7 +50,6 @@ def experiment_run(self_module, i): if not lst: return {'return': 1, 'error': 'No scripts were found'} - # Process each artifact for artifact in sorted(lst, key=lambda x: x.meta.get('alias', '')): meta, script_path = artifact.meta, artifact.path @@ -58,11 +58,10 @@ def experiment_run(self_module, i): 'alias', ''), meta.get( 'uid', '') - # Execute the experiment script mlc_script_input = { 'action': 'run', 'target': 'script' - } + } if exp: for key in exp: ii = {**mlc_script_input, **run_input} @@ -73,7 +72,8 @@ def experiment_run(self_module, i): if r['return'] > 0: return r elif isinstance(exp[key], dict): - return {'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} + return { + 'return': 1, 'error': 'Dictionary inputs are not supported for mlc experiment script'} else: ii[key] = exp[key] r = self_module.action_object.access(ii) @@ -82,7 +82,6 @@ def experiment_run(self_module, i): return {'return': 0} -from collections import defaultdict def parse_value(val): if isinstance(val, list): @@ -108,6 +107,7 @@ def parse_value(val): return val + def convert_input(input_dict): output = defaultdict(dict) @@ -119,4 +119,3 @@ def convert_input(input_dict): output[key].update({k: parse_value(v) for k, v in value.items()}) return dict(output) - From 907cf811895e74b283af80dd3ca23e40debd4866 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 17 May 2025 01:15:33 +0100 Subject: [PATCH 3/3] Update test-mlc-script-features.yml --- .../workflows/test-mlc-script-features.yml | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index 8280cdf53..a511faa89 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -50,18 +50,15 @@ jobs: mlcr get,wkhtmltopdf --quiet - name: Test versions - continue-on-error: true if: runner.os == 'linux' run: | mlcr get,generic-python-lib,_package.scipy --version=1.9.3 --quiet test $? -eq 0 || exit $? mlcr get,generic-python-lib,_package.scipy --version=1.9.2 --quiet test $? -eq 0 || exit $? - mlc find cache --tags=get,generic-python-lib,_package.scipy,version-1.9.3 + mlc find cache --tags=get,generic-python-lib,_package.scipy,version-1.9.2 test $? -eq 0 || exit $? - # Need to add find cache here - # mlcr get,generic-python-lib,_package.scipy --version=1.9.3 --quiet --only_execute_from_cache=True - # test $? -eq 0 || exit 0 + - name: Test python install from src run: | @@ -94,6 +91,34 @@ jobs: run: | mlcr run,docker,container --adr.compiler.tags=gcc --docker_mlc_repo=mlcommons@mlperf-automations --docker_mlc_repo_branch=dev --image_name=mlc-script-app-image-classification-onnx-py --env.MLC_DOCKER_RUN_SCRIPT_TAGS=app,image-classification,onnx,python --env.MLC_DOCKER_IMAGE_BASE=ubuntu:22.04 --env.MLC_DOCKER_IMAGE_REPO=local --quiet + test_experiment: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Configure git longpaths (Windows) + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + - name: Pull MLOps repository + run: | + pip install mlcflow + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + - name: Test mlc experiment script + run: | + mlc experiment script --tags=detect,os --quiet --exp.repeat,=1,2,3 + mlc experiment script --tags=detect,cpu --quiet --exp.explore=2:10:2 + test_mlperf_retinanet_cpp_venv: runs-on: ubuntu-latest strategy: