diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 5bbec09b8..500a9dfdc 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -1,11 +1,8 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: MLPerf inference MLCommons C++ ResNet50 on: - pull_request: - branches: [ "main", "dev", "mlperf-inference" ] + pull_request_target: + branches: [ "main", "dev" ] paths: - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml' - '**' diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 4b7413d97..75e9fe6f9 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -58,27 +58,30 @@ jobs: if: matrix.os != 'windows-latest' run: | mlcr --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - - name: Randomly Execute Step - id: random-check + # Step for Linux/MacOS + - name: Randomly Execute Step (Linux/MacOS) + if: runner.os != 'Windows' run: | - if [[ "$RUNNER_OS" == "Windows" ]]; then - $RANDOM_NUMBER = Get-Random -Maximum 10 - Write-Host "Random number is $RANDOM_NUMBER" - if ($RANDOM_NUMBER -eq 0) { - Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append - } else { - Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append - } + RANDOM_NUMBER=$((RANDOM % 10)) + echo "Random number is $RANDOM_NUMBER" + if [ "$RANDOM_NUMBER" -eq 0 ]; then + echo "run_step=true" >> $GITHUB_ENV else - RANDOM_NUMBER=$((RANDOM % 10)) - echo "Random number is $RANDOM_NUMBER" - if [ "$RANDOM_NUMBER" -eq 0 ]; then - echo "run_step=true" >> $GITHUB_ENV - else - echo "run_step=false" >> $GITHUB_ENV - fi + echo "run_step=false" >> $GITHUB_ENV fi + # Step for Windows + - name: Randomly Execute Step (Windows) + if: runner.os == 'Windows' + run: | + $RANDOM_NUMBER = Get-Random -Maximum 10 + Write-Host "Random number is $RANDOM_NUMBER" + if ($RANDOM_NUMBER -eq 0) { + Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append + } else { + Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append + } + - name: Retrieve secrets from Keeper if: github.repository_owner == 'mlcommons' && env.run_step == 'true' id: ksecrets diff --git a/.github/workflows/test-nvidia-mlperf-inference-implementations.yml b/.github/workflows/test-nvidia-mlperf-inference-implementations.yml index 30f74e605..d05079335 100644 --- a/.github/workflows/test-nvidia-mlperf-inference-implementations.yml +++ b/.github/workflows/test-nvidia-mlperf-inference-implementations.yml @@ -2,7 +2,7 @@ name: MLPerf Inference Nvidia implementations on: schedule: - - cron: "45 17 * * *" #to be adjusted + - cron: "25 23 * * *" #to be adjusted jobs: run_nvidia: diff --git a/automation/cache/README-extra.md b/automation/cache/README-extra.md deleted file mode 100644 index 84d274179..000000000 --- a/automation/cache/README-extra.md +++ /dev/null @@ -1,71 +0,0 @@ -[ [Back to index](../../../docs/README.md) ] - -# CM "cache" automation - -*We suggest you to check [CM introduction](https://github.com/mlcommons/ck/blob/master/docs/introduction-cm.md) - and [CM CLI/API](https://github.com/mlcommons/ck/blob/master/docs/interface.md) to understand CM motivation and concepts.* - -## CM script CLI - -Whenever a [given CM script]() caches the output, you can find it - -Whenever a [CM script](https://access.cknowledge.org/playground/?action=scripts) -caches its output (such as downloaded model or pre-processed data set or built code), -you can find it using the CM "cache" automation as follows: - -```bash -cm show cache -``` - -You can prune cache entries by tags and variations: -```bash -cm show cache --tags=ml-model -cm show cache --tags=python -``` - -You can find a path to a given cache artifact as follows: -```bash -cm find cache --tags=ml-model,bert -``` - -You can delete one or more cache artifacts as follows: -```bash -cm rm cache --tags=ml-model -``` - -You can skip user prompt by adding `-f` flag as follows: -```bash -cm rm cache --tags=ml-model -f -``` - -You can clean the whole cache as follows: -```bash -cm rm cache -f -``` - -## CM python API - -You can access the same functionality via CM Python API as follows: - -```python - -import cmind - -output = cmind.access({'action':'show', - 'automation':'cache,541d6f712a6b464e'}) - -if output['return']>0: - cmind.error(output) - -artifacts = output['list'] - -for artifact in artifacts: - print ('') - print (artifact.path) - print (artifact.meta) - -``` - -## Related - -* [CM "script" automation](../script/README-extra.md) diff --git a/automation/cache/README.md b/automation/cache/README.md deleted file mode 100644 index 0a3114d3b..000000000 --- a/automation/cache/README.md +++ /dev/null @@ -1,87 +0,0 @@ -*This README is automatically generated - don't edit! See [extra README](README-extra.md) for extra notes!* - -### Automation actions - -#### test - - * CM CLI: ```cm test cache``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L15)) - * CM CLI with UID: ```cm test cache,541d6f712a6b464e``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L15)) - * CM Python API: - ```python - import cmind - - r=cm.access({ - 'action':'test' - 'automation':'cache,541d6f712a6b464e' - 'out':'con' - ``` - [add keys from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L15) - ```python - }) - if r['return']>0: - print(r['error']) - ``` - -#### show - - * CM CLI: ```cm show cache``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L54)) - * CM CLI with UID: ```cm show cache,541d6f712a6b464e``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L54)) - * CM Python API: - ```python - import cmind - - r=cm.access({ - 'action':'show' - 'automation':'cache,541d6f712a6b464e' - 'out':'con' - ``` - [add keys from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L54) - ```python - }) - if r['return']>0: - print(r['error']) - ``` - -#### search - - * CM CLI: ```cm search cache``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L153)) - * CM CLI with UID: ```cm search cache,541d6f712a6b464e``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L153)) - * CM Python API: - ```python - import cmind - - r=cm.access({ - 'action':'search' - 'automation':'cache,541d6f712a6b464e' - 'out':'con' - ``` - [add keys from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L153) - ```python - }) - if r['return']>0: - print(r['error']) - ``` - -#### copy_to_remote - - * CM CLI: ```cm copy_to_remote cache``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L186)) - * CM CLI with UID: ```cm copy_to_remote cache,541d6f712a6b464e``` ([add flags (dict keys) from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L186)) - * CM Python API: - ```python - import cmind - - r=cm.access({ - 'action':'copy_to_remote' - 'automation':'cache,541d6f712a6b464e' - 'out':'con' - ``` - [add keys from this API](https://github.com/mlcommons/ck/tree/master/cm-mlops/automation/cache/module.py#L186) - ```python - }) - if r['return']>0: - print(r['error']) - ``` - -### Maintainers - -* [Open MLCommons taskforce on automation and reproducibility](https://cKnowledge.org/mlcommons-taskforce) \ No newline at end of file diff --git a/automation/cache/meta.json b/automation/cache/meta.json deleted file mode 100644 index ac383f937..000000000 --- a/automation/cache/meta.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "alias": "cache", - "automation_alias": "automation", - "automation_uid": "bbeb15d8f0a944a4", - "desc": "Caching cross-platform CM scripts", - "developers": "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)", - "sort": 900, - "tags": [ - "automation" - ], - "uid": "541d6f712a6b464e" -} diff --git a/automation/cache/module.py b/automation/cache/module.py deleted file mode 100644 index b205b539f..000000000 --- a/automation/cache/module.py +++ /dev/null @@ -1,236 +0,0 @@ -import os - -from cmind.automation import Automation -from cmind import utils - - -class CAutomation(Automation): - """ - Automation actions - """ - - ############################################################ - def __init__(self, cmind, automation_file): - super().__init__(cmind, __file__) - - ############################################################ - def test(self, i): - """ - Test automation - - Args: - (CM input dict): - - (out) (str): if 'con', output to console - - automation (str): automation as CM string object - - parsed_automation (list): prepared in CM CLI or CM access function - [ (automation alias, automation UID) ] or - [ (automation alias, automation UID), (automation repo alias, automation repo UID) ] - - (artifact) (str): artifact as CM string object - - (parsed_artifact) (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - ... - - Returns: - (CM return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - * Output from this automation action - - """ - - import json - print(json.dumps(i, indent=2)) - - return {'return': 0} - - ############################################################ - def show(self, i): - """ - Show cache - - Args: - (CM input dict): - - (out) (str): if 'con', output to console - - (env) (bool): if True, show env from cm-cached-state.json - ... - - Returns: - (CM return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - * Output from this automation action - - """ - import json - - # Check parsed automation - if 'parsed_automation' not in i: - return {'return': 1, 'error': 'automation is not specified'} - - console = i.get('out') == 'con' - - show_env = i.get('env', False) - -# Moved to search function -# # Check simplified CMD: cm show cache "get python" -# # If artifact has spaces, treat them as tags! -# artifact = i.get('artifact','') -# tags = i.get('tags','').strip() -# if ' ' in artifact or ',' in artifact: -# del(i['artifact']) -# if 'parsed_artifact' in i: del(i['parsed_artifact']) -# -# new_tags = artifact.replace(' ',',') -# tags = new_tags if tags=='' else new_tags+','+tags -# -# i['tags'] = tags - - # Find CM artifact(s) - i['out'] = None - r = self.search(i) - - if r['return'] > 0: - return r - - lst = r['list'] - for artifact in sorted(lst, key=lambda x: sorted(x.meta['tags'])): - # for artifact in lst: - path = artifact.path - meta = artifact.meta - dependent_cached_path = meta.get( - 'dependent_cached_path', '') - if dependent_cached_path and not os.path.exists( - dependent_cached_path): - continue - - original_meta = artifact.original_meta - - alias = meta.get('alias', '') - uid = meta.get('uid', '') - - tags = meta.get('tags', []) - tags1 = sorted([x for x in tags if not x.startswith('_')]) - tags2 = sorted([x for x in tags if x.startswith('_')]) - tags = tags1 + tags2 - - version = meta.get('version', '') - - if console: - print('') -# print ('* UID: {}'.format(uid)) - print('* Tags: {}'.format(','.join(tags))) - print(' Path: {}'.format(path)) - if version != '': - print(' Version: {}'.format(version)) - - if show_env and console: - path_to_cached_state_file = os.path.join( - path, 'cm-cached-state.json') - - if os.path.isfile(path_to_cached_state_file): - r = utils.load_json(file_name=path_to_cached_state_file) - if r['return'] > 0: - return r - - # Update env and state from cache! - cached_state = r['meta'] - - new_env = cached_state.get('new_env', {}) - if len(new_env) > 0: - print(' New env:') - print( - json.dumps( - new_env, - indent=6, - sort_keys=True).replace( - '{', - '').replace( - '}', - '')) - - new_state = cached_state.get('new_state', {}) - if len(new_state) > 0: - print(' New state:') - print(json.dumps(new_env, indent=6, sort_keys=True)) - - return {'return': 0, 'list': lst} - - ############################################################ - def search(self, i): - """ - Overriding the automation search function to add support for a simplified CMD with tags with spaces - - TBD: add input/output description - """ - # Check simplified CMD: cm show cache "get python" - # If artifact has spaces, treat them as tags! - artifact = i.get('artifact', '') - tags = i.get('tags', '') - - # Tags may be a list (if comes internally from CM scripts) or string if - # comes from CMD - if not isinstance(tags, list): - tags = tags.strip() - - if ' ' in artifact: # or ',' in artifact: - del (i['artifact']) - if 'parsed_artifact' in i: - del (i['parsed_artifact']) - - new_tags = artifact.replace(' ', ',') - tags = new_tags if tags == '' else new_tags + ',' + tags - - i['tags'] = tags - - # Force automation when reruning access with processed input - i['automation'] = 'cache,541d6f712a6b464e' - i['action'] = 'search' - # Avoid recursion - use internal CM add function to add the script - # artifact - i['common'] = True - - # Find CM artifact(s) - return self.cmind.access(i) - - ############################################################ - - def copy_to_remote(self, i): - """ - Add CM automation. - - Args: - (CM input dict): - - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (repos) (str): list of repositories to search for automations (internal & mlcommons@ck by default) - - (output_dir) (str): output directory (./ by default) - - Returns: - (CM return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - """ - - return utils.call_internal_module( - self, __file__, 'module_misc', 'copy_to_remote', i) diff --git a/automation/cache/module_misc.py b/automation/cache/module_misc.py deleted file mode 100644 index d5895edd4..000000000 --- a/automation/cache/module_misc.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -from cmind import utils - - -############################################################ -def copy_to_remote(i): - """ - Add CM automation. - - Args: - (CM input dict): - - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (repos) (str): list of repositories to search for automations (internal & mlcommons@ck by default) - - (output_dir) (str): output directory (./ by default) - - Returns: - (CM return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - """ - - self_module = i['self_module'] - - remote_host = i.get('remote_host') - if not remote_host: - return {'return': 1, - 'error': 'Please input remote host_name/IP via --remote_host'} - remote_mlc_repos_location = i.get( - 'remote_mlc_repos_location', os.path.join( - "/home", os.getlogin(), "CM", "repos")) - remote_mlc_cache_location = os.path.join( - remote_mlc_repos_location, "local", "cache") - - remote_port = i.get('remote_port', '22') - remote_user = i.get('remote_user', os.getlogin()) - - tag_string = i['tags'] - tag_string += ",-tmp" - - mlc_input = {'action': 'show', - 'automation': 'cache', - 'tags': f'{tag_string}', - 'quiet': True - } - r = self_module.cmind.access(mlc_input) - if r['return'] > 0: - return r - - if len(r['list']) == 0: - pass # fixme - elif len(r['list']) > 1: - print("Multiple cache entries found: ") - for k in sorted(r['list'], key=lambda x: x.meta.get('alias', '')): - print(k.path) - x = input("Would you like to copy them all? Y/n: ") - if x.lower() == 'n': - return {'return': 0} - - import json - - for k in sorted(r['list'], key=lambda x: x.meta.get('alias', '')): - path = k.path - cacheid = os.path.basename(path) - - copy_cmd = f"rsync -avz --exclude cm-cached-state.json -e 'ssh -p {remote_port}' {path} {remote_user}@{remote_host}:{remote_mlc_cache_location}" - print(copy_cmd) - os.system(copy_cmd) - - mlc_cached_state_json_file = os.path.join(path, "cm-cached-state.json") - if not os.path.exists(mlc_cached_state_json_file): - return {'return': 1, - 'error': f'cm-cached-state.json file missing in {path}'} - - with open(mlc_cached_state_json_file, "r") as f: - mlc_cached_state = json.load(f) - - new_env = mlc_cached_state['new_env'] - new_state = mlc_cached_state['new_state'] # Todo fix new state - mlc_repos_path = os.environ.get( - 'MLC_REPOS', os.path.join( - os.path.expanduser("~"), "CM", "repos")) - mlc_cache_path = os.path.realpath( - os.path.join(mlc_repos_path, "local", "cache")) - - for key, val in new_env.items(): - - -if isinstance(val, if ) new_env[key] = val.replace( - mlc_cache_path, remote_mlc_cache_location) - - with open("tmp_remote_cached_state.json", "w") as f: - json.dump(mlc_cached_state, f, indent=2) - - remote_cached_state_file_location = os.path.join( - remote_mlc_cache_location, cacheid, "cm-cached-state.json") - copy_cmd = f"rsync -avz -e 'ssh -p {remote_port}' tmp_remote_cached_state.json {remote_user}@{remote_host}:{remote_cached_state_file_location}" - print(copy_cmd) - os.system(copy_cmd) - - return {'return': 0} diff --git a/automation/script/README.md b/automation/script/README.md index bbedf887d..397f19c35 100644 --- a/automation/script/README.md +++ b/automation/script/README.md @@ -1,28 +1,24 @@ -# CM "script" automation specification +# "Script" automation specification -Please check the [CM documentation](https://github.com/mlcommons/ck/tree/master/docs#collective-mind-language-cm) -for more details about the CM automation language. +## Getting started with script automation +* A script is identified by a set of tags and by unique ID. +* Further each script can have multiple variations and they are identified by variation tags which are treated in the same way as tags and identified by a `_` prefix. -## Getting started with CM scripts - -* A CM script is identified by a set of tags and by unique ID. -* Further each CM script can have multiple variations and they are identified by variation tags which are treated in the same way as tags and identified by a `_` prefix. - -### CM script execution flow -* When a CM script is invoked (either by tags or by unique ID), its `_cm.json` is processed first which will check for any `deps` script and if there are, then they are executed in order. +### MLC script execution flow +* When a script is invoked (either by tags or by unique ID), its `meta.yaml` is processed first which will check for any `deps` script and if there are, then they are executed in order. * Once all the `deps` scripts are executed, `customize.py` file is checked and if existing `preprocess` function inside it is executed if present. -* Then any `prehook_deps` CM scripts mentioned in `_cm.json` are executed similar to `deps` +* Then any `prehook_deps` scripts mentioned in `meta.yaml` are executed similar to `deps` * After this, keys in `env` dictionary is exported as `ENV` variables and `run` file if exists is executed. -* Once run file execution is done, any `posthook_deps` CM scripts mentioned in `_cm.json` are executed similar to `deps` +* Once run file execution is done, any `posthook_deps` scripts mentioned in `meta.yaml` are executed similar to `deps` * Then `postprocess` function inside customize.py is executed if present. -* After this stage any `post_deps` CM scripts mentioned in `_cm.json` is executed. +* After this stage any `post_deps` scripts mentioned in `meta.yaml` is executed. ** If a script is already cached, then the `preprocess`, `run file` and `postprocess` executions won't happen and only the dependencies marked as `dynamic` will be executed from `deps`, `prehook_deps`, `posthook_deps` and `postdeps`. ### Input flags -When we run a CM script we can also pass inputs to it and any input added in `input_mapping` dictionary inside `_cm.json` gets converted to the corresponding `ENV` variable. +When we run an MLC script we can also pass inputs to it and any input added in `input_mapping` dictionary inside `meta.yaml` gets converted to the corresponding `ENV` variable. ### Conditional execution of any `deps`, `post_deps` We can use `skip_if_env` dictionary inside any `deps`, `prehook_deps`, `posthook_deps` or `post_deps` to make its execution conditional @@ -33,7 +29,7 @@ We can specify any specific version of a script using `version`. `version_max` a * When `version_max` is given, any version below this if present in the cache or detected in the system can be chosen. If nothing is detected `default_version` if present and if below `version_max` will be used for installation. Otherwise `version_max_usable` (additional needed input for `version_max`) will be used as `version`. ### Variations -* Variations are used to customize CM script and each unique combination of variations uses a unique cache entry. Each variation can turn on `env` keys also any other meta including dependencies specific to it. Variations are turned on like tags but with a `_` prefix. For example, if a script is having tags `"get,myscript"`, to call the variation `"test"` inside it, we have to use tags `"get,myscript,_test"`. +* Variations are used to customize scripts and each unique combination of variations uses a unique cache entry. Each variation can turn on `env` keys also any other meta including dependencies specific to it. Variations are turned on like tags but with a `_` prefix. For example, if a script is having tags `"get,myscript"`, to call the variation `"test"` inside it, we have to use tags `"get,myscript,_test"`. #### Variation groups `group` is a key to map variations into a group and at any time only one variation from a group can be used in the variation tags. For example, both `cpu` and `cuda` can be two variations under the `device` group, but user can at any time use either `cpu` or `cuda` as variation tags but not both. @@ -41,8 +37,7 @@ We can specify any specific version of a script using `version`. `version_max` a #### Dynamic variations Sometimes it is difficult to add all variations needed for a script like say `batch_size` which can take many different values. To handle this case, we support dynamic variations using '#' where '#' can be dynamically replaced by any string. For example, `"_batch_size.8"` can be used as a tag to turn on the dynamic variation `"_batch_size.#"`. -### ENV flow during CM script execution -* [TBD] Issue added [here](https://github.com/mlcommons/ck/issues/382) +### ENV flow during MLC script execution * During a given script execution incoming `env` dictionary is saved `(saved_env)` and all the updates happens on a copy of it. * Once a script execution is over (which includes all the dependent script executions as well), newly created keys and any updated keys are merged with the `saved_env` provided the keys are mentioned in `new_env_keys` * Same behaviour applies to `state` dictionary. diff --git a/automation/script/assets/scripts-workflow.png b/automation/script/assets/scripts-workflow.png deleted file mode 100644 index 60d0ef715..000000000 Binary files a/automation/script/assets/scripts-workflow.png and /dev/null differ diff --git a/automation/script/module_help.py b/automation/script/module_help.py deleted file mode 100644 index 820378180..000000000 --- a/automation/script/module_help.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -from cmind import utils - -# Pring help about script - - -def print_help(i): - - meta = i.get('meta', '') - path = i.get('path', '') - - if len(meta) == 0 and path == '': - return {'return': 0} - - print('') - print( - 'Help for this CM script ({},{}):'.format( - meta.get( - 'alias', ''), meta.get( - 'uid', ''))) - - print('') - print('Path to this automation recipe: {}'.format(path)) - - variations = meta.get('variations', {}) - if len(variations) > 0: - print('') - print('Available variations:') - print('') - for v in sorted(variations): - print(' _' + v) - - input_mapping = meta.get('input_mapping', {}) - if len(input_mapping) > 0: - print('') - print('Available flags mapped to environment variables:') - print('') - for k in sorted(input_mapping): - v = input_mapping[k] - - print(' --{} -> --env.{}'.format(k, v)) - - input_description = meta.get('input_description', {}) - if len(input_description) > 0: - # Check if has important ones (sort) - sorted_keys = [] - all_keys = sorted(list(input_description.keys())) - - for k in sorted( - all_keys, key=lambda x: input_description[x].get('sort', 0)): - v = input_description[k] - if v.get('sort', 0) > 0: - sorted_keys.append(k) - - print('') - print('Available flags (Python API dict keys):') - print('') - for k in all_keys: - v = input_description[k] - n = v.get('desc', '') - - x = ' --' + k - if n != '': - x += ' ({})'.format(n) - - print(x) - - if len(sorted_keys) > 0: - print('') - print('Main flags:') - print('') - for k in sorted_keys: - v = input_description[k] - n = v.get('desc', '') - - x = ' --' + k - - d = None - if 'default' in v: - d = v.get('default', '') - - if d is not None: - x += '=' + d - - c = v.get('choices', []) - if len(c) > 0: - x += ' {' + ','.join(c) + '}' - - if n != '': - x += ' ({})'.format(n) - - print(x) - - print('') - x = input('Would you like to see a Python API with a list of common keys/flags for all scripts including this one (y/N)? ') - - x = x.strip().lower() - - skip_delayed_help = False if x in ['y', 'yes'] else True - - r = {'return': 0} - - if skip_delayed_help: - r['skip_delayed_help'] = True - - return r diff --git a/automation/script/template-ae-python/README-extra.md b/automation/script/template-ae-python/README-extra.md deleted file mode 100644 index 05e53dc1a..000000000 --- a/automation/script/template-ae-python/README-extra.md +++ /dev/null @@ -1,2 +0,0 @@ -# CM script to run and reproduce experiments - diff --git a/automation/script/template-ae-python/_cm.yaml b/automation/script/template-ae-python/_cm.yaml deleted file mode 100644 index 261e4cf75..000000000 --- a/automation/script/template-ae-python/_cm.yaml +++ /dev/null @@ -1,38 +0,0 @@ -cache: false - -deps: - # Detect host OS features - - tags: detect,os - - # Detect/install python - - tags: get,python - names: - - python - - python3 - -script_name: run - -input_mapping: - experiment: MLC_EXPERIMENT - -default_env: - MLC_EXPERIMENT: '1' - -variations: - install_deps: - script_name: install_deps - - run: - script_name: run - - reproduce: - script_name: reproduce - - plot: - script_name: plot - - analyze: - script_name: analyze - - validate: - script_name: validate diff --git a/automation/script/template-ae-python/analyze.bat b/automation/script/template-ae-python/analyze.bat deleted file mode 100644 index 375cfaebf..000000000 --- a/automation/script/template-ae-python/analyze.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -rem echo. -rem %MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -rem IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-ae-python/analyze.sh b/automation/script/template-ae-python/analyze.sh deleted file mode 100644 index 53c10c73c..000000000 --- a/automation/script/template-ae-python/analyze.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -#echo "" -#${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -#test $? -eq 0 || exit 1 diff --git a/automation/script/template-ae-python/install_deps.bat b/automation/script/template-ae-python/install_deps.bat deleted file mode 100644 index 3419d9511..000000000 --- a/automation/script/template-ae-python/install_deps.bat +++ /dev/null @@ -1,18 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -if exist "%MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt" ( - - echo. - echo Installing requirements.txt ... - echo. - - %MLC_PYTHON_BIN_WITH_PATH% -m pip install -r %MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt - IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% -) diff --git a/automation/script/template-ae-python/install_deps.sh b/automation/script/template-ae-python/install_deps.sh deleted file mode 100644 index 5e8c50a20..000000000 --- a/automation/script/template-ae-python/install_deps.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -if test -f "${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt"; then - echo "" - echo "Installing requirements.txt ..." - echo "" - - ${MLC_PYTHON_BIN_WITH_PATH} -m pip install -r ${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt - test $? -eq 0 || exit 1 -fi diff --git a/automation/script/template-ae-python/main.py b/automation/script/template-ae-python/main.py deleted file mode 100644 index 48b974b7f..000000000 --- a/automation/script/template-ae-python/main.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -if __name__ == "__main__": - - print('') - print('Main script:') - print('Experiment: {}'.format(os.environ.get('MLC_EXPERIMENT', ''))) - print('') - - exit(0) diff --git a/automation/script/template-ae-python/plot.bat b/automation/script/template-ae-python/plot.bat deleted file mode 100644 index 375cfaebf..000000000 --- a/automation/script/template-ae-python/plot.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -rem echo. -rem %MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -rem IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-ae-python/plot.sh b/automation/script/template-ae-python/plot.sh deleted file mode 100644 index 53c10c73c..000000000 --- a/automation/script/template-ae-python/plot.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -#echo "" -#${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -#test $? -eq 0 || exit 1 diff --git a/automation/script/template-ae-python/reproduce.bat b/automation/script/template-ae-python/reproduce.bat deleted file mode 100644 index 375cfaebf..000000000 --- a/automation/script/template-ae-python/reproduce.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -rem echo. -rem %MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -rem IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-ae-python/reproduce.sh b/automation/script/template-ae-python/reproduce.sh deleted file mode 100644 index 53c10c73c..000000000 --- a/automation/script/template-ae-python/reproduce.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -#echo "" -#${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -#test $? -eq 0 || exit 1 diff --git a/automation/script/template-ae-python/run.bat b/automation/script/template-ae-python/run.bat deleted file mode 100644 index f1b69d26d..000000000 --- a/automation/script/template-ae-python/run.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -echo. -%MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-ae-python/run.sh b/automation/script/template-ae-python/run.sh deleted file mode 100644 index a4b86e69a..000000000 --- a/automation/script/template-ae-python/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -echo "" -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -test $? -eq 0 || exit 1 diff --git a/automation/script/template-ae-python/validate.bat b/automation/script/template-ae-python/validate.bat deleted file mode 100644 index 375cfaebf..000000000 --- a/automation/script/template-ae-python/validate.bat +++ /dev/null @@ -1,12 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV MLC_EXPERIMENT: %MLC_EXPERIMENT% - -rem echo. -rem %MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -rem IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-ae-python/validate.sh b/automation/script/template-ae-python/validate.sh deleted file mode 100644 index 53c10c73c..000000000 --- a/automation/script/template-ae-python/validate.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV MLC_EXPERIMENT: ${MLC_EXPERIMENT}" - -#echo "" -#${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -#test $? -eq 0 || exit 1 diff --git a/automation/script/template-python/README-extra.md b/automation/script/template-python/README-extra.md deleted file mode 100644 index 582991f6d..000000000 --- a/automation/script/template-python/README-extra.md +++ /dev/null @@ -1 +0,0 @@ -# CM script diff --git a/automation/script/template-python/_cm.yaml b/automation/script/template-python/_cm.yaml deleted file mode 100644 index 11f646860..000000000 --- a/automation/script/template-python/_cm.yaml +++ /dev/null @@ -1,23 +0,0 @@ -cache: false - -deps: - # Detect host OS features - - tags: detect,os - - # Detect/install python - - tags: get,python - names: - - python - - python3 - -input_mapping: - var1: MLC_VAR1 - req: PIP_REQUIREMENTS - -default_env: - MLC_VAR1: 'something' - -variations: - req: - env: - PIP_REQUIREMENTS: True diff --git a/automation/script/template-python/customize.py b/automation/script/template-python/customize.py deleted file mode 100644 index 8961ab5ca..000000000 --- a/automation/script/template-python/customize.py +++ /dev/null @@ -1,32 +0,0 @@ -from cmind import utils -import os - - -def preprocess(i): - - print('') - print('Preprocessing ...') - - os_info = i['os_info'] - - env = i['env'] - - meta = i['meta'] - - automation = i['automation'] - - quiet = (env.get('MLC_QUIET', False) == 'yes') - - print(' ENV MLC_VAR1: {}'.format(env.get('MLC_VAR1', ''))) - - return {'return': 0} - - -def postprocess(i): - - print('') - print('Postprocessing ...') - - env = i['env'] - - return {'return': 0} diff --git a/automation/script/template-python/main.py b/automation/script/template-python/main.py deleted file mode 100644 index 68245e7bd..000000000 --- a/automation/script/template-python/main.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - -if __name__ == "__main__": - - print('') - print('Main script:') - print('ENV MLC_VAR1: {}'.format(os.environ.get('MLC_VAR1', ''))) - print('') - - exit(0) diff --git a/automation/script/template-python/requirements.txt b/automation/script/template-python/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/automation/script/template-python/run.bat b/automation/script/template-python/run.bat deleted file mode 100644 index 11e897362..000000000 --- a/automation/script/template-python/run.bat +++ /dev/null @@ -1,25 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV PIP_REQUIREMENTS: %PIP_REQUIREMENTS% -echo ENV MLC_VAR1: %MLC_VAR1% - -if "%PIP_REQUIREMENTS%" == "True" ( - if exist "%MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt" ( - - echo. - echo Installing requirements.txt ... - echo. - - %MLC_PYTHON_BIN_WITH_PATH% -m pip install -r %MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt - IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% - ) -) - -echo. -%MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-python/run.sh b/automation/script/template-python/run.sh deleted file mode 100644 index a3e2021b9..000000000 --- a/automation/script/template-python/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV PIP_REQUIREMENTS: ${PIP_REQUIREMENTS}" -echo "ENV MLC_VAR1: ${MLC_VAR1}" - -if [ "${PIP_REQUIREMENTS}" == "True" ]; then - if test -f "${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt"; then - echo "" - echo "Installing requirements.txt ..." - echo "" - - ${MLC_PYTHON_BIN_WITH_PATH} -m pip install -r ${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt - test $? -eq 0 || exit 1 - fi -fi - -echo "" -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -test $? -eq 0 || exit 1 diff --git a/automation/script/template-pytorch/README-extra.md b/automation/script/template-pytorch/README-extra.md deleted file mode 100644 index 582991f6d..000000000 --- a/automation/script/template-pytorch/README-extra.md +++ /dev/null @@ -1 +0,0 @@ -# CM script diff --git a/automation/script/template-pytorch/_cm.yaml b/automation/script/template-pytorch/_cm.yaml deleted file mode 100644 index 22cd7a635..000000000 --- a/automation/script/template-pytorch/_cm.yaml +++ /dev/null @@ -1,42 +0,0 @@ -cache: false - -deps: - # Detect host OS features - - tags: detect,os - - # Detect/install python - - tags: get,python - names: - - python - - python3 - - - tags: get,generic-python-lib,_torch - skip_if_env: - USE_CUDA: - - yes - - - tags: get,generic-python-lib,_torch_cuda - enable_if_env: - USE_CUDA: - - yes - - - tags: get,generic-python-lib,_package.numpy - - -input_mapping: - var1: MLC_VAR1 - req: PIP_REQUIREMENTS - -default_env: - MLC_VAR1: 'something' - -variations: - req: - env: - PIP_REQUIREMENTS: True - - cuda: - env: - USE_CUDA: yes - deps: - - tags: get,cuda diff --git a/automation/script/template-pytorch/customize.py b/automation/script/template-pytorch/customize.py deleted file mode 100644 index 8961ab5ca..000000000 --- a/automation/script/template-pytorch/customize.py +++ /dev/null @@ -1,32 +0,0 @@ -from cmind import utils -import os - - -def preprocess(i): - - print('') - print('Preprocessing ...') - - os_info = i['os_info'] - - env = i['env'] - - meta = i['meta'] - - automation = i['automation'] - - quiet = (env.get('MLC_QUIET', False) == 'yes') - - print(' ENV MLC_VAR1: {}'.format(env.get('MLC_VAR1', ''))) - - return {'return': 0} - - -def postprocess(i): - - print('') - print('Postprocessing ...') - - env = i['env'] - - return {'return': 0} diff --git a/automation/script/template-pytorch/main.py b/automation/script/template-pytorch/main.py deleted file mode 100644 index 3bfcd7572..000000000 --- a/automation/script/template-pytorch/main.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - -import torch - -if __name__ == "__main__": - - print('') - print('Main script:') - print('ENV MLC_VAR1: {}'.format(os.environ.get('MLC_VAR1', ''))) - print('ENV USE_CUDA: {}'.format(os.environ.get('USE_CUDA', ''))) - print('') - print('PyTorch version: {}'.format(torch.__version__)) - print('') - - exit(0) diff --git a/automation/script/template-pytorch/requirements.txt b/automation/script/template-pytorch/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/automation/script/template-pytorch/run.bat b/automation/script/template-pytorch/run.bat deleted file mode 100644 index 11e897362..000000000 --- a/automation/script/template-pytorch/run.bat +++ /dev/null @@ -1,25 +0,0 @@ -@echo off - -set CUR_DIR=%cd% - -echo. -echo Current execution path: %CUR_DIR% -echo Path to script: %MLC_TMP_CURRENT_SCRIPT_PATH% -echo ENV PIP_REQUIREMENTS: %PIP_REQUIREMENTS% -echo ENV MLC_VAR1: %MLC_VAR1% - -if "%PIP_REQUIREMENTS%" == "True" ( - if exist "%MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt" ( - - echo. - echo Installing requirements.txt ... - echo. - - %MLC_PYTHON_BIN_WITH_PATH% -m pip install -r %MLC_TMP_CURRENT_SCRIPT_PATH%\requirements.txt - IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% - ) -) - -echo. -%MLC_PYTHON_BIN_WITH_PATH% %MLC_TMP_CURRENT_SCRIPT_PATH%\main.py -IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/automation/script/template-pytorch/run.sh b/automation/script/template-pytorch/run.sh deleted file mode 100644 index a3e2021b9..000000000 --- a/automation/script/template-pytorch/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -CUR_DIR=${PWD} - -echo "" -echo "Current execution path: ${CUR_DIR}" -echo "Path to script: ${MLC_TMP_CURRENT_SCRIPT_PATH}" -echo "ENV PIP_REQUIREMENTS: ${PIP_REQUIREMENTS}" -echo "ENV MLC_VAR1: ${MLC_VAR1}" - -if [ "${PIP_REQUIREMENTS}" == "True" ]; then - if test -f "${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt"; then - echo "" - echo "Installing requirements.txt ..." - echo "" - - ${MLC_PYTHON_BIN_WITH_PATH} -m pip install -r ${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt - test $? -eq 0 || exit 1 - fi -fi - -echo "" -${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/main.py -test $? -eq 0 || exit 1 diff --git a/automation/script/template/README-extra.md b/automation/script/template/README-extra.md deleted file mode 100644 index 582991f6d..000000000 --- a/automation/script/template/README-extra.md +++ /dev/null @@ -1 +0,0 @@ -# CM script diff --git a/automation/script/template/customize.py b/automation/script/template/customize.py deleted file mode 100644 index bd7c12dd3..000000000 --- a/automation/script/template/customize.py +++ /dev/null @@ -1,24 +0,0 @@ -from cmind import utils -import os - - -def preprocess(i): - - os_info = i['os_info'] - - env = i['env'] - - meta = i['meta'] - - automation = i['automation'] - - quiet = (env.get('MLC_QUIET', False) == 'yes') - - return {'return': 0} - - -def postprocess(i): - - env = i['env'] - - return {'return': 0} diff --git a/automation/script/template_list_of_scripts.md b/automation/script/template_list_of_scripts.md deleted file mode 100644 index 07fb95cb7..000000000 --- a/automation/script/template_list_of_scripts.md +++ /dev/null @@ -1,52 +0,0 @@ -[ [Back to index](README.md) ] - - - -This is an automatically generated list of portable and reusable automation recipes (CM scripts) -with a [human-friendly interface (CM)](https://github.com/mlcommons/ck) -to run a growing number of ad-hoc MLPerf, MLOps, and DevOps scripts -from [MLCommons projects](https://github.com/mlcommons/cm4mlops/tree/main/script) -and [research papers](https://www.youtube.com/watch?v=7zpeIVwICa4) -in a unified way on any operating system with any software and hardware -natively or inside containers. - -Click on any automation recipe below to learn how to run and reuse it -via CM command line, Python API or GUI. - -CM scripts can easily chained together into automation workflows using `deps` and `tags` keys -while automatically updating all environment variables and paths -for a given task and platform [using simple JSON or YAML](https://github.com/mlcommons/ck/blob/master/mlc-mlops/script/app-image-classification-onnx-py/_cm.yaml). - - -*Note that CM is a community project being developed and extended by [MLCommons members and individual contributors](../CONTRIBUTING.md) - - you can find source code of CM scripts maintained by MLCommons [here](../mlc-mlops/script). - Please join [Discord server](https://discord.gg/JjWNWXKxwT) to participate in collaborative developments or provide your feedback.* - - -# License - -[Apache 2.0](LICENSE.md) - - -# Copyright - -2022-2024 [MLCommons](https://mlcommons.org) - - - - - -# List of CM scripts by categories - -{{MLC_TOC_CATEGORIES}} - -{{MLC_TOC2}} - -# List of all sorted CM scripts - -{{MLC_TOC}} - - -{{MLC_MAIN}} diff --git a/automation/script/template-ae-python/customize.py b/automation/script/templates/default/customize.py similarity index 100% rename from automation/script/template-ae-python/customize.py rename to automation/script/templates/default/customize.py diff --git a/automation/script/template/run.bat b/automation/script/templates/default/run.bat similarity index 100% rename from automation/script/template/run.bat rename to automation/script/templates/default/run.bat diff --git a/automation/script/template/run.sh b/automation/script/templates/default/run.sh similarity index 100% rename from automation/script/template/run.sh rename to automation/script/templates/default/run.sh