From 4596e8de03d14aa8bcfd6a1b7c38eef70ff964af Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 8 Mar 2025 23:24:51 +0000 Subject: [PATCH 01/14] Test mlc tests --- script/run-mlperf-inference-mobilenet-models/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/run-mlperf-inference-mobilenet-models/meta.yaml b/script/run-mlperf-inference-mobilenet-models/meta.yaml index 813b1ef93..209c58a6d 100644 --- a/script/run-mlperf-inference-mobilenet-models/meta.yaml +++ b/script/run-mlperf-inference-mobilenet-models/meta.yaml @@ -159,6 +159,7 @@ variations: group: optimization use-neon: alias: neon + tests: run_inputs: - env: From 0416deb709106532bd8e1f8cb54c470ea0db50f7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 06:01:09 +0530 Subject: [PATCH 02/14] Fix test script --- automation/script/module.py | 263 ++---------------------------------- 1 file changed, 12 insertions(+), 251 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 731d3b075..49b610f1c 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -502,8 +502,7 @@ def _run(self, i): ii = {} ii['tags'] = tags_string ii['out'] = None - for key in ["automation", "parsed_automation", - "artifact", "parsed_artifact"]: + for key in ["automation", "artifact", "item", "details"]: if i.get(key): ii[key] = i[key] @@ -2792,13 +2791,19 @@ def search(self, i): # Find MLC script(s) based on thier tags to get their meta (can be more than 1) # Then check if variations exists inside meta - i['tags'] = ','.join(script_tags) + ii = {} + ii['tags'] = ','.join(script_tags) - i['out'] = None - i['common'] = True + ii['out'] = None + ii['common'] = True + + ii['target'] = "script" + for key in ["automation", "item", + "artifact", "details"]: + if i.get(key): + ii[key] = i[key] - i['target_name'] = "script" - r = super(ScriptAutomation, self).search(i) + r = super(ScriptAutomation, self).search(ii) if r['return'] > 0: return r @@ -2901,7 +2906,6 @@ def test(self, i): # Find script item(s) i['out'] = None r = self.search(i) - if r['return'] > 0: return r @@ -2910,7 +2914,6 @@ def test(self, i): for script_item in lst: path = script_item.path meta = script_item.meta - original_meta = script_item.original_meta alias = meta.get('alias', '') uid = meta.get('uid', '') @@ -3097,248 +3100,6 @@ def native_run(self, i): return {'return': 0, 'return_code': rc} - ############################################################ - def add(self, i): - """ - Add MLC script - - Args: - (MLC input dict): - - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in MLC CLI or MLC access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (tags) (str): tags to find an MLC script (MLC artifact) - - (script_name) (str): name of script (it will be copied to the new entry and added to the meta) - - (tags) (string or list): tags to be added to meta - - (new_tags) (string or list): new tags to be added to meta (the same as tags) - - (json) (bool): if True, record JSON meta instead of YAML - - (meta) (dict): preloaded meta - - (template) (string): template to use (python) - (python) (bool): template=python - (pytorch) (bool): template=pytorch - ... - - Returns: - (MLC return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - """ - - import shutil - - console = i.get('out') == 'con' - logger = self.action_object.logger - - # Try to find script artifact by alias and/or tags - # ii = utils.sub_input(i, self.cmind.cfg['artifact_keys']) - ii = {} - ii['tags'] = tags_string - ii['out'] = None - - for key in ["automation", "parsed_automation", - "artifact", "parsed_artifact"]: - if i.get(key): - ii[key] = i[key] - - parsed_artifact = i.get('parsed_artifact', []) - - artifact_obj = parsed_artifact[0] if len(parsed_artifact) > 0 else None - artifact_repo = parsed_artifact[1] if len( - parsed_artifact) > 1 else None - - script_name = '' - if 'script_name' in i: - script_name = i.get('script_name', '').strip() - del (i['script_name']) - - if script_name != '' and not os.path.isfile(script_name): - return {'return': 1, - 'error': 'file {} not found'.format(script_name)} - - # Move tags from input to meta of the newly created script artifact - res = utils.convert_tags_to_list(i['tags']) - if res['return'] > 0: - return res - tags_list = res['tags'] - if 'tags' in i: - del (i['tags']) - - if len(tags_list) == 0: - if console: - x = input( - 'Please specify a combination of unique tags separated by comma for this script: ') - x = x.strip() - if x != '': - tags_list = x.split(',') - - if len(tags_list) == 0: - return { - 'return': 1, 'error': 'you must specify a combination of unique tags separate by comman using "--new_tags"'} - - # Add placeholder (use common action) - ii['out'] = 'con' - # Avoid recursion - use internal MLC add function to add the script - # artifact - ii['common'] = True - - # Check template path - template_dir = 'template' - - template = i.get('template', '') - - if template == '': - if i.get('python', False): - template = 'python' - elif i.get('pytorch', False): - template = 'pytorch' - - if template != '': - template_dir += '-' + template - - template_path = os.path.join(self.path, template_dir) - - if not os.path.isdir(template_path): - return {'return': 1, 'error': 'template path {} not found'.format( - template_path)} - - # Check if preloaded meta exists - meta = { - 'cache': False - # 20240127: Grigori commented that because newly created script meta looks ugly - # 'new_env_keys':[], - # 'new_state_keys':[], - # 'input_mapping':{}, - # 'docker_input_mapping':{}, - # 'deps':[], - # 'prehook_deps':[], - # 'posthook_deps':[], - # 'post_deps':[], - # 'versions':{}, - # 'variations':{}, - # 'input_description':{} - } - - fmeta = os.path.join( - template_path, - self.action_object.cfg['file_cmeta']) - - r = utils.load_yaml_and_json(fmeta) - if r['return'] == 0: - utils.merge_dicts({'dict1': meta, - 'dict2': r['meta'], - 'append_lists': True, - 'append_unique': True}) - - # Check meta from CMD - xmeta = i.get('meta', {}) - - if len(xmeta) > 0: - utils.merge_dicts({'dict1': meta, 'dict2': xmeta, - 'append_lists': True, 'append_unique': True}) - - meta['automation_alias'] = self.meta['alias'] - meta['automation_uid'] = self.meta['uid'] - meta['tags'] = tags_list - - script_name_base = script_name - script_name_ext = '' - if script_name != '': - # separate name and extension - j = script_name.rfind('.') - if j >= 0: - script_name_base = script_name[:j] - script_name_ext = script_name[j:] - - meta['script_name'] = script_name_base - - ii['meta'] = meta - ii['action'] = 'add' - - use_yaml = True if not i.get('json', False) else False - - if use_yaml: - ii['yaml'] = True - - ii['automation'] = 'script,5b4e0237da074764' - - for k in ['parsed_automation', 'parsed_artifact']: - if k in ii: - del ii[k] - - if artifact_repo is not None: - ii['artifact'] = utils.assemble_object2( - artifact_repo) + ':' + utils.assemble_object2(artifact_obj) - - r_obj = self.action_object.access(ii) - if r_obj['return'] > 0: - return r_obj - - new_script_path = r_obj['path'] - - if console: - logger.info('Created script in {}'.format(new_script_path)) - - # Copy files from template (only if exist) - files = [ - (template_path, 'README-extra.md', ''), - (template_path, 'customize.py', ''), - (template_path, 'main.py', ''), - (template_path, 'requirements.txt', ''), - (template_path, 'install_deps.bat', ''), - (template_path, 'install_deps.sh', ''), - (template_path, 'plot.bat', ''), - (template_path, 'plot.sh', ''), - (template_path, 'analyze.bat', ''), - (template_path, 'analyze.sh', ''), - (template_path, 'validate.bat', ''), - (template_path, 'validate.sh', '') - ] - - if script_name == '': - files += [(template_path, 'run.bat', ''), - (template_path, 'run.sh', '')] - else: - if script_name_ext == '.bat': - files += [(template_path, 'run.sh', script_name_base + '.sh')] - files += [('', script_name, script_name)] - - else: - files += [(template_path, 'run.bat', - script_name_base + '.bat')] - files += [('', script_name, script_name_base + '.sh')] - - for x in files: - path = x[0] - f1 = x[1] - f2 = x[2] - - if f2 == '': - f2 = f1 - - if path != '': - f1 = os.path.join(path, f1) - - if os.path.isfile(f1): - f2 = os.path.join(new_script_path, f2) - - if console: - logger.info(' * Copying {} to {}'.format(f1, f2)) - - shutil.copyfile(f1, f2) - - return r_obj ########################################################################## def _get_name_for_dynamic_variation_tag(script, variation_tag): From b037415b5c9915cf043bc59e5144d692c60fbe51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 9 Mar 2025 00:31:34 +0000 Subject: [PATCH 03/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 49b610f1c..e18c81f40 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3100,8 +3100,8 @@ def native_run(self, i): return {'return': 0, 'return_code': rc} - ########################################################################## + def _get_name_for_dynamic_variation_tag(script, variation_tag): ''' Returns the variation name in meta for the dynamic_variation_tag From ed164cd52c0cbf081edbb5f3489d9a76375280af Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 00:45:36 +0000 Subject: [PATCH 04/14] Update check-broken-links.yml --- .github/workflows/check-broken-links.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/check-broken-links.yml b/.github/workflows/check-broken-links.yml index 06034c32a..8a192b29a 100644 --- a/.github/workflows/check-broken-links.yml +++ b/.github/workflows/check-broken-links.yml @@ -13,7 +13,6 @@ jobs: steps: - uses: actions/checkout@v4 - # Checks the status of hyperlinks in .md files in verbose mode - name: Check links uses: gaurav-nelson/github-action-markdown-link-check@v1 From 7d16f3c52301208055076feeaab1274413a51dcb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 06:53:32 +0530 Subject: [PATCH 05/14] Fix test script --- automation/script/module.py | 6 +++--- script/run-mlperf-inference-mobilenet-models/meta.yaml | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index e18c81f40..555329a90 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2973,14 +2973,14 @@ def test(self, i): if given_variations: v_split = [] run_variations = [] - for i, v in enumerate(given_variations): + for v in given_variations: v_split = v.split(",") - for t in v_split: + for index, t in enumerate(v_split): if not t.startswith("_"): # variations must begin with _. We # support both with and without _ # in the meta - given_variations[i] = f"_{t}" + v_split[index] = f"_{t}" if v_split: run_variations.append( ",".join(v_split)) diff --git a/script/run-mlperf-inference-mobilenet-models/meta.yaml b/script/run-mlperf-inference-mobilenet-models/meta.yaml index 209c58a6d..f942aaaf4 100644 --- a/script/run-mlperf-inference-mobilenet-models/meta.yaml +++ b/script/run-mlperf-inference-mobilenet-models/meta.yaml @@ -164,9 +164,8 @@ tests: run_inputs: - env: MLC_TEST_ONE_RUN: 'yes' - variations_list: - - tflite - - accuracy_only + variations_list: + - tflite,accuracy-only adr: compiler: tags: gcc From c948a47cbeb4f804102b963210d62d2fc79eab27 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 07:02:39 +0530 Subject: [PATCH 06/14] Fix unzip version detection --- script/get-generic-sys-util/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index a3faa4254..1dea058c9 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -693,7 +693,7 @@ variations: unzip: env: MLC_SYS_UTIL_NAME: unzip - MLC_SYS_UTIL_VERSION_CMD: unzip --version + MLC_SYS_UTIL_VERSION_CMD: unzip -v new_env_keys: - MLC_UNZIP_VERSION state: From 9b78d2218a9fdebbda7c8518e3b239f2d43c2be0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 07:19:17 +0530 Subject: [PATCH 07/14] Update bazel version for tensorflow build from src --- script/install-tensorflow-from-src/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/install-tensorflow-from-src/meta.yaml b/script/install-tensorflow-from-src/meta.yaml index c219d1179..5697128d3 100644 --- a/script/install-tensorflow-from-src/meta.yaml +++ b/script/install-tensorflow-from-src/meta.yaml @@ -56,7 +56,7 @@ versions: version_max_usable: 17.0.6 version_min: 17.0.6 - tags: get,bazel - version: 6.5.0 + version: 7.4.1 env: MLC_GIT_CHECKOUT: master v1.15.0: From 08b7bf4e53139ea7b9f236145984a09d3ec53d8f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 07:24:48 +0530 Subject: [PATCH 08/14] Remove copyright file in template script --- script/template-script/COPYRIGHT.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 script/template-script/COPYRIGHT.md diff --git a/script/template-script/COPYRIGHT.md b/script/template-script/COPYRIGHT.md deleted file mode 100644 index 9e44ad290..000000000 --- a/script/template-script/COPYRIGHT.md +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Notice - -© 2022-2025 MLCommons. All Rights Reserved. - -This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: - -[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) - -Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. From 6b62fb9e29ad1924a9ec067a17ba261e4a3e954a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 07:57:33 +0530 Subject: [PATCH 09/14] Use short run for tflite test --- .github/scripts/process_individual_tests.py | 2 ++ script/run-mlperf-inference-mobilenet-models/meta.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/scripts/process_individual_tests.py b/.github/scripts/process_individual_tests.py index a8df24239..708a80963 100644 --- a/.github/scripts/process_individual_tests.py +++ b/.github/scripts/process_individual_tests.py @@ -35,3 +35,5 @@ ret = mlc.access(ii) if ret['return'] > 0: raise Exception(r['error']) + ii = {'action': 'rm', 'target': 'cache', 'f': True} + ret = mlc.access(ii) diff --git a/script/run-mlperf-inference-mobilenet-models/meta.yaml b/script/run-mlperf-inference-mobilenet-models/meta.yaml index f942aaaf4..1fd77a9de 100644 --- a/script/run-mlperf-inference-mobilenet-models/meta.yaml +++ b/script/run-mlperf-inference-mobilenet-models/meta.yaml @@ -169,3 +169,5 @@ tests: adr: compiler: tags: gcc + imagenet-preprocessed: + tags: _500 From 22fe2b46857d840f47046a1b98474fbe9a36908b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 08:26:33 +0530 Subject: [PATCH 10/14] Change tflite test to find-performance --- script/run-mlperf-inference-mobilenet-models/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-mlperf-inference-mobilenet-models/meta.yaml b/script/run-mlperf-inference-mobilenet-models/meta.yaml index 1fd77a9de..2eb6647b3 100644 --- a/script/run-mlperf-inference-mobilenet-models/meta.yaml +++ b/script/run-mlperf-inference-mobilenet-models/meta.yaml @@ -165,7 +165,7 @@ tests: - env: MLC_TEST_ONE_RUN: 'yes' variations_list: - - tflite,accuracy-only + - tflite,find-performance adr: compiler: tags: gcc From a9b3fd2921b77d952fe1eb1a8abe52ea176707df Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 03:04:02 +0000 Subject: [PATCH 11/14] Update test-mlc-script-features.yml --- .github/workflows/test-mlc-script-features.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index a61f6b3c7..88708f577 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -44,7 +44,10 @@ jobs: mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NCHW mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NHWC # Tests run-scripts on windows,ubuntu,macos - mlcr get,wkhtmltopdf + + - name: Test native script name variants + run: | + mlcr get,wkhtmltopdf --quiet - name: Test versions continue-on-error: true From 034796b3abb6d92444e1d0abec6143d20f2fbadd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 16:18:35 +0530 Subject: [PATCH 12/14] Added docker_prune option for mlc test script --- automation/script/module.py | 15 +++++++++++++-- script/get-generic-sys-util/meta.yaml | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 555329a90..666ce04ce 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3020,12 +3020,14 @@ def test(self, i): logger.info(ii) r = self.action_object.access(ii) if r['return'] > 0: - return r - + return r + if is_true(i.get('docker_prune','')): + docker_prune() return {'return': 0, 'list': lst} ############################################################ + def native_run(self, i): """ Add MLC script @@ -4861,6 +4863,15 @@ def _update_env(env, key=None, value=None): return {'return': 0} +def docker_prune(): + try: + # Run the docker prune command with -a (removes all unused images, not just dangling ones) + result = subprocess.run(["docker", "system", "prune", "-a", "-f"], + capture_output=True, text=True, check=True) + print("Docker prune output:\n", result.stdout) + except subprocess.CalledProcessError as e: + print("Error while running Docker prune:\n", e.stderr) + ########################################################################## def update_env_with_values(env, fail_on_not_found=False, extra_env=None): diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index 1dea058c9..41ed35745 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -42,12 +42,14 @@ tests: fail_safe: 'yes' ignore_missing: 'yes' test-all-variations: 'yes' + docker_prune: 'yes' - docker: 'yes' docker_os: ubuntu docker_os_version: '24.04' fail_safe: 'yes' ignore_missing: 'yes' test-all-variations: 'yes' + docker_prune: 'yes' uid: bb0393afa8404a11 variations: cmake: From 2d0e3c21f0860a5a6a67367bb0b13f1bf25c57dc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 9 Mar 2025 10:49:01 +0000 Subject: [PATCH 13/14] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 666ce04ce..641382707 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3020,14 +3020,13 @@ def test(self, i): logger.info(ii) r = self.action_object.access(ii) if r['return'] > 0: - return r - if is_true(i.get('docker_prune','')): + return r + if is_true(i.get('docker_prune', '')): docker_prune() return {'return': 0, 'list': lst} ############################################################ - def native_run(self, i): """ Add MLC script @@ -4863,9 +4862,11 @@ def _update_env(env, key=None, value=None): return {'return': 0} + def docker_prune(): try: - # Run the docker prune command with -a (removes all unused images, not just dangling ones) + # Run the docker prune command with -a (removes all unused images, not + # just dangling ones) result = subprocess.run(["docker", "system", "prune", "-a", "-f"], capture_output=True, text=True, check=True) print("Docker prune output:\n", result.stdout) From ce55980cf17eaac4415dc3e0e37fc4e637e7f281 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 10:51:11 +0000 Subject: [PATCH 14/14] Update check-broken-links.yml --- .github/workflows/check-broken-links.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/check-broken-links.yml b/.github/workflows/check-broken-links.yml index 8a192b29a..0a07b08ec 100644 --- a/.github/workflows/check-broken-links.yml +++ b/.github/workflows/check-broken-links.yml @@ -1,5 +1,4 @@ name: "Check .md README files for broken links" - on: pull_request: branches: