From 8f03e5e8d3151b2339f51f8870ab8e6dee704c6b Mon Sep 17 00:00:00 2001 From: Arjun Date: Thu, 27 Mar 2025 18:19:33 +0530 Subject: [PATCH 01/17] Support aocc download with EULA --- automation/script/module.py | 3 +++ script/download-file/meta.yaml | 3 +++ script/extract-file/customize.py | 4 ++-- script/extract-file/meta.yaml | 3 +++ script/get-aocc/customize.py | 7 +++++++ script/get-aocc/meta.yaml | 22 ++++++++++++++++++++++ 6 files changed, 40 insertions(+), 2 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 1325f5197..02db3571c 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4727,6 +4727,9 @@ def find_cached_script(i): r = docker_utils.get_container_path_script(i) if not os.path.exists(r['value_env']): # Need to rm this cache entry + logger.debug( + recursion_spaces + + ' - Skipping cached entry as the dependent path {} is missing!'.format(r['value_env'])) skip_cached_script = True continue diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index 832275c1c..6f940186e 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -6,6 +6,7 @@ can_force_cache: true category: DevOps automation default_env: MLC_RCLONE_COPY_USING: sync + MLC_EXTRACT_REMOVE_EXTRACTED: no deps: - tags: detect,os - enable_if_env: @@ -40,6 +41,8 @@ tags_help: download file uid: 9cdc8dc41aae437e variations: cmutil: + alias: mlcutil + mlcutil: default: true env: MLC_DOWNLOAD_TOOL: cmutil diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index d55f8787c..1d7a11c29 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -1,7 +1,7 @@ from mlc import utils import os import hashlib - +from utils import * def preprocess(i): @@ -209,7 +209,7 @@ def postprocess(i): env['MLC_GET_DEPENDENT_CACHED_PATH'] = filepath # Check if need to remove archive after extraction - if env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '').lower() != 'no': + if is_true(env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '')): archive_filepath = env.get('MLC_EXTRACT_FILEPATH', '') if archive_filepath != '' and os.path.isfile(archive_filepath): os.remove(archive_filepath) diff --git a/script/extract-file/meta.yaml b/script/extract-file/meta.yaml index 56f29fe1d..7e69437e1 100644 --- a/script/extract-file/meta.yaml +++ b/script/extract-file/meta.yaml @@ -36,9 +36,12 @@ tags_help: extract file uid: 3f0b76219d004817 variations: keep: + group: keep + default: true env: MLC_EXTRACT_REMOVE_EXTRACTED: 'no' no-remove-extracted: + group: keep env: MLC_EXTRACT_REMOVE_EXTRACTED: 'no' path.#: diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 5c8b33712..0759e8d05 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -25,6 +25,13 @@ def preprocess(i): aocc_path = env['MLC_AOCC_DIR_PATH'] if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')): env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') + else: + for l in os.listdir(aocc_path): + if os.path.exists(os.path.join(aocc_path, l, 'bin', 'clang')): + aocc_path = os.path.join(aocc_path, l) + env['MLC_AOCC_DIR_PATH'] = aocc_path + env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') + r = i['automation'].find_artifact({'file_name': exe_c, 'env': env, diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index e1ffa999c..41ff05210 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -6,10 +6,24 @@ category: Compiler automation clean_files: [] deps: - tags: detect,os +- tags: download,file,_mlcutil + extra_cache_tags: aocc,download + update_tags_from_env_with_prefix: + _url.: + - MLC_AOCC_URL + force_cache: true + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_AOCC_TAR_FILE_PATH + enable_if_env: + MLC_AOCC_URL: + - on + MLC_AOCC_ACCEPT_EULA: + - on - tags: extract,file update_tags_from_env_with_prefix: _path.: - MLC_AOCC_TAR_FILE_PATH + extra_cache_tags: aocc,extract force_cache: true env: MLC_EXTRACT_FINAL_ENV_NAME: MLC_AOCC_DIR_PATH @@ -21,6 +35,7 @@ deps: input_mapping: tar_file_path: MLC_AOCC_TAR_FILE_PATH aocc_dir: MLC_AOCC_DIR_PATH + accept_eula: MLC_AOCC_ACCEPT_EULA name: Detect or install AOCC compiler new_env_keys: @@ -43,6 +58,13 @@ tags: - get - aocc uid: 1ceb0656e99a44ec +versions: + 5.0.0: + env: + MLC_AOCC_URL: https://download.amd.com/developer/eula/aocc/aocc-5-0/aocc-compiler-5.0.0.tar + MLC_DOWNLOAD_CHECKSUM: 966fac2d2c759e9de6e969c10ada7a7b306c113f7f1e07ea376829ec86380daa + MLC_AOCC_NEEDS_TAR: yes + MLC_VERSION: '5.0.0-Build#1377' variations: path.#: env: From 0f2e60952525010c79a1a3be33af67c72f548a3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 27 Mar 2025 12:51:11 +0000 Subject: [PATCH 02/17] [Automated Commit] Format Codebase [skip ci] --- script/extract-file/customize.py | 1 + script/get-aocc/customize.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index 1d7a11c29..c7e50be3a 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -3,6 +3,7 @@ import hashlib from utils import * + def preprocess(i): variation_tags = i.get('variation_tags', []) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 0759e8d05..1e42e360a 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -27,11 +27,11 @@ def preprocess(i): env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') else: for l in os.listdir(aocc_path): - if os.path.exists(os.path.join(aocc_path, l, 'bin', 'clang')): + if os.path.exists(os.path.join( + aocc_path, l, 'bin', 'clang')): aocc_path = os.path.join(aocc_path, l) env['MLC_AOCC_DIR_PATH'] = aocc_path env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') - r = i['automation'].find_artifact({'file_name': exe_c, 'env': env, From d74431026faa07d396f486c16e0c283a7e64537b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 12:52:26 +0000 Subject: [PATCH 03/17] Update build_wheel.yml --- .github/workflows/build_wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 3b7919325..5467a7cc9 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -8,6 +8,7 @@ on: - VERSION jobs: + build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel From 490ece62ace1e9bb2638957431306e98fc91b6af Mon Sep 17 00:00:00 2001 From: Arjun Date: Thu, 27 Mar 2025 23:35:13 +0530 Subject: [PATCH 04/17] Fix get-gcc,get-oneapi variations --- script/get-gcc/meta.yaml | 4 ++-- script/get-one-api/meta.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-gcc/meta.yaml b/script/get-gcc/meta.yaml index 4de1812f5..d8d7827b3 100644 --- a/script/get-gcc/meta.yaml +++ b/script/get-gcc/meta.yaml @@ -31,6 +31,6 @@ tags: - get-gcc uid: dbf4ab5cbed74372 variations: - _path.#: + path.#: env: - MLC_GCC_DIR_PATH: # + MLC_GCC_DIR_PATH: '#' diff --git a/script/get-one-api/meta.yaml b/script/get-one-api/meta.yaml index 1033eb772..47cf218b0 100644 --- a/script/get-one-api/meta.yaml +++ b/script/get-one-api/meta.yaml @@ -26,6 +26,6 @@ tags: - get-oneapi uid: 1af872e81ef54742 variations: - _path.#: + path.#: env: MLC_ONEAPI_DIR_PATH: "#" From 52a79a089f387d4173e09f991d4b0e39ba033b52 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 28 Mar 2025 00:46:33 +0530 Subject: [PATCH 05/17] Add tests for mlc-scripts installation (#323) --- .github/workflows/test-mlc-script-pip.yml | 59 +++++++++++++++++++++++ setup.py | 20 +++++--- 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/test-mlc-script-pip.yml diff --git a/.github/workflows/test-mlc-script-pip.yml b/.github/workflows/test-mlc-script-pip.yml new file mode 100644 index 000000000..6e93af9c5 --- /dev/null +++ b/.github/workflows/test-mlc-script-pip.yml @@ -0,0 +1,59 @@ +name: Test pip installation for mlc-script + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlc-script-pip.yml' + - '**' + - '!**.md' + +jobs: + test_mlc_script_install_pypi: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: windows-latest + - os: macos-latest + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: install mlc-scripts + run: | + pip install mlc-scripts + - name: run detect-os script + run: | + mlcr detect-os -j + + test_mlc_script_install_pull_request_source: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: windows-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: install mlc-scripts from source repo + run: | + pip install . + - name: run detect-os script + run: | + mlcr detect-os -j diff --git a/setup.py b/setup.py index 0ba25f32d..311c51537 100644 --- a/setup.py +++ b/setup.py @@ -21,25 +21,28 @@ def get_project_meta(file_path="pyproject.toml"): dict: Dictionary containing the project metadata. """ try: - with open(file_path, "rb") as f: + with open(file_path, "r") as f: + content_tmp = f.read() if 'tomllib' in globals(): - project_data = tomllib.load(f) # Use tomllib for Python 3.11+ + # Use tomllib for Python 3.11+ + project_data = tomllib.loads(content_tmp) else: - project_data = toml.load(f) # Use toml for older versions + # Use toml for older versions + project_data = toml.loads(content_tmp) # Extract metadata under [project] project_meta = project_data.get("project", {}) return project_meta except FileNotFoundError: - print(f"Error: {file_path} not found.") + raise FileNotFoundError(f"Error: {file_path} not found.") except Exception as e: - print(f"Error reading {file_path}: {e}") + raise RuntimeError(f"Error reading {file_path}: {e}") return {} def check_prerequisites(): - """Check if Git and python-venv are installed on the system.""" + """Check if Git and python-venv are installed on the system. """ try: # Check for Git subprocess.run(["git", "--version"], check=True, @@ -77,9 +80,10 @@ def run(self): 'branch': branch, # 'checkout': commit_hash }) - print(res) + if res['return'] > 0: - return res['return'] + raise Exception( + f"Return code:{res['return']} with error:{res.get('error')}") # subprocess.run(["echo", "Custom command executed!"], check=True) except Exception as e: From 40d6e849b0268c7905827251cfec1aa369768f08 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 28 Mar 2025 03:01:19 +0530 Subject: [PATCH 06/17] Cleanup aocc installation, added test --- script/download-file/customize.py | 23 +++++++++++------------ script/download-file/meta.yaml | 2 +- script/download-file/run.bat | 2 +- script/download-file/run.sh | 6 +++--- script/get-aocc/customize.py | 24 ++++++++++++++++++++++++ script/get-aocc/meta.yaml | 17 ++++++++++++++++- 6 files changed, 56 insertions(+), 18 deletions(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index f525c440f..767646b47 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -76,7 +76,7 @@ def preprocess(i): print('') print('Downloading from {}'.format(url)) - if '&' in url and tool != "cmutil": + if '&' in url and tool != "mlcutil": if os_info['platform'] == 'windows': url = '"' + url + '"' else: @@ -112,8 +112,8 @@ def preprocess(i): else: env['MLC_DOWNLOAD_FILENAME'] = "index.html" - if tool == "cmutil": - cmutil_require_download = 0 + if tool == "mlcutil": + mlcutil_require_download = 0 if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '': if os_info['platform'] == 'windows': checksum_cmd = f"cd {q}{filepath}{q} {xsep} md5sum -c{x_c} {x}{escape_special_chars(env['MLC_DOWNLOAD_CHECKSUM_FILE'])}" @@ -126,7 +126,7 @@ def preprocess(i): text=True, shell=True, env=subprocess_env) - elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '': + elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '' and os.path.isfile(env['MLC_DOWNLOAD_FILENAME']): if os_info['platform'] == 'windows': checksum_cmd = f"echo {env.get('MLC_DOWNLOAD_CHECKSUM')} {x}{escape_special_chars(env['MLC_DOWNLOAD_FILENAME'])} | md5sum -c{x_c} -" else: @@ -137,8 +137,8 @@ def preprocess(i): text=True, shell=True, env=subprocess_env) - if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get( - 'MLC_DOWNLOAD_CHECKSUM', '') != '': + if (env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get( + 'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.isfile(env['MLC_DOWNLOAD_FILENAME']): # print(checksum_result) #for debugging if "checksum did not match" in checksum_result.stderr.lower(): computed_checksum = subprocess.run( @@ -155,10 +155,10 @@ def preprocess(i): except PermissionError: return { "return": 1, "error": f"Permission denied to delete file {env['MLC_DOWNLOAD_FILENAME']}."} - cmutil_require_download = 1 + mlcutil_require_download = 1 elif "no such file" in checksum_result.stderr.lower(): - # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. Downloading through cmutil.") - cmutil_require_download = 1 + # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. Downloading through mlcutil.") + mlcutil_require_download = 1 elif checksum_result.returncode > 0: return { "return": 1, "error": f"Error while checking checksum: {checksum_result.stderr}"} @@ -166,10 +166,9 @@ def preprocess(i): print( f"File {env['MLC_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..") else: - cmutil_require_download = 1 + mlcutil_require_download = 1 - if cmutil_require_download == 1: - cm = automation.action_object + if mlcutil_require_download == 1: for i in range(1, 5): r = download_file({ 'url': url, diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index 6f940186e..28ac8e31b 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -45,7 +45,7 @@ variations: mlcutil: default: true env: - MLC_DOWNLOAD_TOOL: cmutil + MLC_DOWNLOAD_TOOL: mlcutil group: download-tool curl: default_env: diff --git a/script/download-file/run.bat b/script/download-file/run.bat index dcd7603c9..b72d580d1 100644 --- a/script/download-file/run.bat +++ b/script/download-file/run.bat @@ -19,7 +19,7 @@ if not "%MLC_DOWNLOAD_LOCAL_FILE_PATH%" == "" ( set require_download=0 ) -if "%MLC_DOWNLOAD_TOOL%" == "cmutil" ( +if "%MLC_DOWNLOAD_TOOL%" == "mlcutil" ( set require_download=0 ) diff --git a/script/download-file/run.sh b/script/download-file/run.sh index b737ea34e..2664ac52b 100644 --- a/script/download-file/run.sh +++ b/script/download-file/run.sh @@ -9,8 +9,8 @@ fi # Assume download is required by default require_download=1 -# No download needed if a local file path is specified or the tool is 'cmutil' -if [[ -n "${MLC_DOWNLOAD_LOCAL_FILE_PATH}" || ${MLC_DOWNLOAD_TOOL} == "cmutil" ]]; then +# No download needed if a local file path is specified or the tool is 'mlcutil' +if [[ -n "${MLC_DOWNLOAD_LOCAL_FILE_PATH}" || ${MLC_DOWNLOAD_TOOL} == "mlcutil" ]]; then require_download=0 fi @@ -50,7 +50,7 @@ if [[ ${require_download} == 1 ]]; then fi # Verify checksum again if necessary -if [[ ${MLC_DOWNLOAD_TOOL} == "cmutil" || ${require_download} == 1 ]]; then +if [[ ${MLC_DOWNLOAD_TOOL} == "mlcutil" || ${require_download} == 1 ]]; then if [[ -n "${MLC_DOWNLOAD_CHECKSUM_CMD}" ]]; then echo -e "\nVerifying checksum after download: ${MLC_DOWNLOAD_CHECKSUM_CMD}" eval "${MLC_DOWNLOAD_CHECKSUM_CMD}" || exit $? diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 1e42e360a..69473f234 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -1,5 +1,23 @@ from mlc import utils import os +from utils import * + + +def ask_url_acceptance(url): + print(f"Please take a moment to read the EULA at this URL:\n{url}") + print("\nDo you accept the terms of this EULA? [yes/no]") + + while True: + response = input().lower() + if response in ["yes", "y"]: + print("You have accepted the EULA.") + return True + elif response in ["no", "n"]: + print("You have not accepted the EULA.") + return False + else: + print("Invalid input. Please enter 'yes' or 'no'.") + def predeps(i): @@ -9,6 +27,12 @@ def predeps(i): if env.get('MLC_AOCC_TAR_FILE_PATH', '') != '': env['MLC_AOCC_NEEDS_TAR'] = 'yes' + elif is_true(env.get('MLC_AOCC_DOWNLOAD')) and not is_true(env.get('MLC_AOCC_ACCEPT_EULA')): + url = "https://www.amd.com/en/developer/aocc/aocc-compiler/eula.html" + accepted = ask_url_acceptance(url) + if accepted: + env['MLC_AOCC_ACCEPT_EULA'] = 'yes' + return {'return': 0} diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index 41ff05210..22adb675c 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -62,10 +62,25 @@ versions: 5.0.0: env: MLC_AOCC_URL: https://download.amd.com/developer/eula/aocc/aocc-5-0/aocc-compiler-5.0.0.tar - MLC_DOWNLOAD_CHECKSUM: 966fac2d2c759e9de6e969c10ada7a7b306c113f7f1e07ea376829ec86380daa + MLC_DOWNLOAD_CHECKSUM: c01ee764a8370a2ed546c1a38e41bb42 MLC_AOCC_NEEDS_TAR: yes MLC_VERSION: '5.0.0-Build#1377' variations: + download-and-install: + group: install-type + default: true + default_version: "5.0.0" + env: + MLC_AOCC_DOWNLOAD: yes + local-install: + group: install-type + env: + MLC_AOCC_DOWNLOAD: no path.#: + default_variation: + install-type: local-install env: MLC_AOCC_DIR_PATH: '#' +tests: + run_inputs: + - accept_eula: yes From d6efb6e13763ebca99ff085b009c48d12842c0c5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 27 Mar 2025 21:31:34 +0000 Subject: [PATCH 07/17] [Automated Commit] Format Codebase [skip ci] --- script/get-aocc/customize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 69473f234..3d1570299 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -19,7 +19,6 @@ def ask_url_acceptance(url): print("Invalid input. Please enter 'yes' or 'no'.") - def predeps(i): os_info = i['os_info'] From f0f4847f21cf72eb60414da0329bb1d1c2d4aa54 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:35:49 +0000 Subject: [PATCH 08/17] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7dea76edb..6d7de6e6a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.0.2 From 67ca733730597e397c4f6dd5b0d14dee49f78e9e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:39:09 +0000 Subject: [PATCH 09/17] Update and rename build_wheel.yml to build_wheel_off.yml --- .github/workflows/{build_wheel.yml => build_wheel_off.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build_wheel.yml => build_wheel_off.yml} (98%) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel_off.yml similarity index 98% rename from .github/workflows/build_wheel.yml rename to .github/workflows/build_wheel_off.yml index 5467a7cc9..a24482b1c 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel_off.yml @@ -1,4 +1,4 @@ -name: Build wheel and release into PYPI +name: Build wheel and release into PYPI (off now) on: push: From ba6914e45653904e18c3ebd8fcd5ae1e1c767900 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 27 Mar 2025 18:39:37 +0000 Subject: [PATCH 10/17] Update and rename build_wheels_on_release.yml to build_wheels.yml --- .../workflows/{build_wheels_on_release.yml => build_wheels.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build_wheels_on_release.yml => build_wheels.yml} (95%) diff --git a/.github/workflows/build_wheels_on_release.yml b/.github/workflows/build_wheels.yml similarity index 95% rename from .github/workflows/build_wheels_on_release.yml rename to .github/workflows/build_wheels.yml index fd62c80c2..708ae35e4 100644 --- a/.github/workflows/build_wheels_on_release.yml +++ b/.github/workflows/build_wheels.yml @@ -1,4 +1,4 @@ -name: Build wheel and release into PYPI +name: Build wheel (only src) and release into PYPI on: release: From 50c5a2e922307d4f8a6466bb2fc997af47db0842 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 28 Mar 2025 00:46:33 +0530 Subject: [PATCH 11/17] Add tests for mlc-scripts installation (#323) --- .github/workflows/test-mlc-script-pip.yml | 59 +++++++++++++++++++++++ setup.py | 20 +++++--- 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/test-mlc-script-pip.yml diff --git a/.github/workflows/test-mlc-script-pip.yml b/.github/workflows/test-mlc-script-pip.yml new file mode 100644 index 000000000..6e93af9c5 --- /dev/null +++ b/.github/workflows/test-mlc-script-pip.yml @@ -0,0 +1,59 @@ +name: Test pip installation for mlc-script + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlc-script-pip.yml' + - '**' + - '!**.md' + +jobs: + test_mlc_script_install_pypi: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: windows-latest + - os: macos-latest + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: install mlc-scripts + run: | + pip install mlc-scripts + - name: run detect-os script + run: | + mlcr detect-os -j + + test_mlc_script_install_pull_request_source: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.8"] + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: windows-latest + - os: macos-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: install mlc-scripts from source repo + run: | + pip install . + - name: run detect-os script + run: | + mlcr detect-os -j diff --git a/setup.py b/setup.py index 0ba25f32d..311c51537 100644 --- a/setup.py +++ b/setup.py @@ -21,25 +21,28 @@ def get_project_meta(file_path="pyproject.toml"): dict: Dictionary containing the project metadata. """ try: - with open(file_path, "rb") as f: + with open(file_path, "r") as f: + content_tmp = f.read() if 'tomllib' in globals(): - project_data = tomllib.load(f) # Use tomllib for Python 3.11+ + # Use tomllib for Python 3.11+ + project_data = tomllib.loads(content_tmp) else: - project_data = toml.load(f) # Use toml for older versions + # Use toml for older versions + project_data = toml.loads(content_tmp) # Extract metadata under [project] project_meta = project_data.get("project", {}) return project_meta except FileNotFoundError: - print(f"Error: {file_path} not found.") + raise FileNotFoundError(f"Error: {file_path} not found.") except Exception as e: - print(f"Error reading {file_path}: {e}") + raise RuntimeError(f"Error reading {file_path}: {e}") return {} def check_prerequisites(): - """Check if Git and python-venv are installed on the system.""" + """Check if Git and python-venv are installed on the system. """ try: # Check for Git subprocess.run(["git", "--version"], check=True, @@ -77,9 +80,10 @@ def run(self): 'branch': branch, # 'checkout': commit_hash }) - print(res) + if res['return'] > 0: - return res['return'] + raise Exception( + f"Return code:{res['return']} with error:{res.get('error')}") # subprocess.run(["echo", "Custom command executed!"], check=True) except Exception as e: From 01588a40bdeeab9d392160351731526ace2c5c11 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 28 Mar 2025 19:55:06 +0000 Subject: [PATCH 12/17] Support Intel SDE Tool, improved individual script tests (#333) * Support subfolder in extract-file * Enable test for intel sde * Dont restrict mlc test to console * Improved github action for individual script tests --- .github/scripts/list_modified_files.py | 36 ++++++++++ .github/workflows/build_wheel_off.yml | 1 - .github/workflows/build_wheels.yml | 1 - .../workflows/run-tests-on-modified-meta.yml | 57 +++++++++++++++ automation/script/module.py | 2 +- script/download-and-extract/meta.yaml | 1 + script/download-file/customize.py | 2 +- script/extract-file/customize.py | 18 ++++- script/extract-file/meta.yaml | 1 + script/get-aocc/customize.py | 1 - script/get-aocc/run.sh | 2 - script/get-intel-sde-tool/customize.py | 71 +++++++++++++++++++ script/get-intel-sde-tool/meta.yaml | 46 ++++++++++++ script/get-intel-sde-tool/run.sh | 6 ++ 14 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/list_modified_files.py create mode 100644 .github/workflows/run-tests-on-modified-meta.yml create mode 100644 script/get-intel-sde-tool/customize.py create mode 100644 script/get-intel-sde-tool/meta.yaml create mode 100644 script/get-intel-sde-tool/run.sh diff --git a/.github/scripts/list_modified_files.py b/.github/scripts/list_modified_files.py new file mode 100644 index 000000000..3a6d4b90f --- /dev/null +++ b/.github/scripts/list_modified_files.py @@ -0,0 +1,36 @@ +import yaml +import sys +import json + + +def get_num_runs(filepath): + with open(filepath, 'r') as file: + content = yaml.safe_load(file) + tests = content.get('tests', {}) + if tests: + num_tests = len(tests.get('run_inputs', [])) + else: + num_tests = 0 + uid = content['uid'] + return uid, num_tests + + +def process_files(files): + filenames = files.split() + return [ + { + "file": file, + "uid": uid, + "num_tests": num_tests + } + for file in filenames + if file.endswith('meta.yaml') and (uid := get_num_runs(file)) + for _, num_tests in [uid] + ] + + +if __name__ == "__main__": + changed_files = sys.stdin.read().strip() + processed_files = process_files(changed_files) + json_processed_files = json.dumps(processed_files) + print(f"::set-output name=modified_files::{json_processed_files}") diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index a24482b1c..4fbc24dba 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -8,7 +8,6 @@ on: - VERSION jobs: - build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 708ae35e4..86b93ac06 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -6,7 +6,6 @@ on: workflow_dispatch: {} jobs: - build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml new file mode 100644 index 000000000..79ecb1198 --- /dev/null +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -0,0 +1,57 @@ +# This workflow will run configured tests for any updated MLC script +name: MLC test script on modified meta + +on: + pull_request: + branches: [ "main", "dev" ] + paths: + - 'script/**meta.yaml' + +jobs: + get-modified-files: + runs-on: ubuntu-latest + outputs: + modified_files: ${{ steps.modified-files.outputs.modified_files }} + + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + + - name: Get changed files + id: modified-files + run: | + git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} + git fetch upstream + changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only) + echo "$changed_files" | python3 .github/scripts/list_modified_files.py + + + + process_modified_files: + runs-on: ubuntu-latest + needs: get_modified_files + strategy: + fail_fast: false + matrix: + file_info: ${{ fromJSON(needs.get_modified_files.outputs.modified_files) }} + num_run: ${{ range(1, fromJSON(matrix.file_info.num_tests) + 1) }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Process meta.yaml file + run: | + echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.num_run }}" + + pip install mlcflow + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet + diff --git a/automation/script/module.py b/automation/script/module.py index 02db3571c..1b4875237 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2929,7 +2929,7 @@ def test(self, i): alias = meta.get('alias', '') uid = meta.get('uid', '') - if console: + if console or True: # Todo restrict to console only? logger.info(path) test_config = meta.get('tests', '') if test_config: diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 1d02bfbae..2b8748b2e 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -19,6 +19,7 @@ input_mapping: new_env_keys: - MLC_DOWNLOAD_DOWNLOADED_PATH* - MLC_EXTRACT_EXTRACTED_PATH +- MLC_EXTRACT_EXTRACTED_SUBDIR_PATH - <<>> - <<>> - <<>> diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 767646b47..8edbf9413 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -138,7 +138,7 @@ def preprocess(i): shell=True, env=subprocess_env) if (env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get( - 'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.isfile(env['MLC_DOWNLOAD_FILENAME']): + 'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.exists(env['MLC_DOWNLOAD_FILENAME']): # print(checksum_result) #for debugging if "checksum did not match" in checksum_result.stderr.lower(): computed_checksum = subprocess.run( diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index c7e50be3a..2a0d99c4b 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -176,11 +176,11 @@ def postprocess(i): extract_to_folder = env.get('MLC_EXTRACT_TO_FOLDER', '') extract_path = env.get('MLC_EXTRACT_PATH', '') - + folderpath = None extracted_file = env.get('MLC_EXTRACT_EXTRACTED_FILENAME', '') # Preparing filepath - # Can be either full extracted filename (such as model) or folder + # Can be either the full extracted filename (such as model file) or folder if extracted_file != '': filename = os.path.basename(extracted_file) @@ -191,8 +191,8 @@ def postprocess(i): filepath = os.path.join(folderpath, filename) else: - filepath = os.getcwd() # Extracted to the root cache folder + folderpath = os.getcwd() if not os.path.exists(filepath): return { @@ -215,6 +215,18 @@ def postprocess(i): if archive_filepath != '' and os.path.isfile(archive_filepath): os.remove(archive_filepath) + # Check if only a single folder is created and if so, export the folder + # name + if folderpath: + sub_items = os.listdir(folderpath) + sub_folders = [ + item for item in sub_items if os.path.isdir( + os.path.join( + folderpath, item))] + if len(sub_folders) == 1: + env['MLC_EXTRACT_EXTRACTED_SUBDIR_PATH'] = os.path.join( + folderpath, sub_folders[0]) + # Since may change directory, check if need to clean some temporal files automation.clean_some_tmp_files({'env': env}) diff --git a/script/extract-file/meta.yaml b/script/extract-file/meta.yaml index 7e69437e1..79f1538c3 100644 --- a/script/extract-file/meta.yaml +++ b/script/extract-file/meta.yaml @@ -23,6 +23,7 @@ input_mapping: to: MLC_EXTRACT_PATH new_env_keys: - MLC_EXTRACT_EXTRACTED_PATH +- MLC_EXTRACT_EXTRACTED_SUBDIR_PATH - <<>> - MLC_GET_DEPENDENT_CACHED_PATH new_state_keys: [] diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 3d1570299..513539438 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -65,7 +65,6 @@ def preprocess(i): 'run_script_input': i['run_script_input'], 'recursion_spaces': i['recursion_spaces']}) if r['return'] > 0: - return r return {'return': 0} diff --git a/script/get-aocc/run.sh b/script/get-aocc/run.sh index fe19b6c8d..6266120bb 100644 --- a/script/get-aocc/run.sh +++ b/script/get-aocc/run.sh @@ -4,5 +4,3 @@ echo "${aocc_bin} --version" ${aocc_bin} --version > tmp-ver.out test $? -eq 0 || exit $? - -cat tmp-ver.out diff --git a/script/get-intel-sde-tool/customize.py b/script/get-intel-sde-tool/customize.py new file mode 100644 index 000000000..021d28eb9 --- /dev/null +++ b/script/get-intel-sde-tool/customize.py @@ -0,0 +1,71 @@ +from mlc import utils +import os +import subprocess + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + exe = 'sde.exe' if os_info['platform'] == 'windows' else 'sde' + + if env.get('MLC_INTEL_SDE_DIR_PATH', '') == '': + if env.get('MLC_EXTRACT_EXTRACTED_SUBDIR_PATH', '') != '': + env['MLC_INTEL_SDE_DIR_PATH'] = env['MLC_EXTRACT_EXTRACTED_SUBDIR_PATH'] + elif env.get('MLC_EXTRACT_EXTRACTED_PATH', '') != '': + env['MLC_INTEL_SDE_DIR_PATH'] = env['MLC_EXTRACT_EXTRACTED_PATH'] + else: + return {'return': 1, 'error': 'Intel SDE path not found'} + + if 'MLC_INTEL_SDE_BIN_WITH_PATH' not in env: + if env.get('MLC_INTEL_SDE_DIR_PATH', '') != '': + sde_path = env['MLC_INTEL_SDE_DIR_PATH'] + if os.path.exists(os.path.join(sde_path, 'sde')): + env['MLC_TMP_PATH'] = sde_path + + r = i['automation'].find_artifact({'file_name': exe, + 'env': env, + 'os_info': os_info, + 'default_path_env_key': 'PATH', + 'detect_version': True, + 'env_path_key': 'MLC_INTEL_SDE_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': i['recursion_spaces']}) + if r['return'] > 0: + return r + + return {'return': 0} + + +def detect_version(i): + r = i['automation'].parse_version({'match_text': r'Software Development Emulator.\s+Version:\s*([\d.]+)', + 'group_number': 1, + 'env_key': 'MLC_INTEL_SDE_VERSION', + 'which_env': i['env']}) + if r['return'] > 0: + return r + version = r['version'] + + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + r = detect_version(i) + if r['return'] > 0: + return r + + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_INTEL_SDE_BIN_WITH_PATH'] + if '+PATH' not in env: + env['+PATH'] = [] + + os_info = i['os_info'] + + return {'return': 0} diff --git a/script/get-intel-sde-tool/meta.yaml b/script/get-intel-sde-tool/meta.yaml new file mode 100644 index 000000000..763593965 --- /dev/null +++ b/script/get-intel-sde-tool/meta.yaml @@ -0,0 +1,46 @@ +alias: get-intel-sde-tool +automation_alias: script +automation_uid: 5b4e0237da074764 +category: MLC Utils +cache: True +deps: + - tags: detect,os + - tags: download-and-extract,_extract + enable_if_env: + MLC_DOWNLOAD_URL: + - on + skip_if_env: + MLC_INTEL_SDE_DIR_PATH: + - on + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + extra_cache_tags: intel,sde,tool + force_cache: True +new_env_keys: + - MLC_INTEL_SDE_DIR_PATH + - MLC_INTEL_SDE_BIN_WITH_PATH + - MLC_INTEL_SDE_VERSION + - +PATH +new_state_keys: [] +post_deps: [] +posthook_deps: [] +tags: +- get +- tool +- sde +- intel +input_mapping: + sde_dir: MLC_INTEL_SDE_DIR_PATH +uid: 3546409e71c847bc +variations: + version.9_53_0: + group: version + default: true + env: + MLC_DOWNLOAD_URL: https://downloadmirror.intel.com/850782/sde-external-9.53.0-2025-03-16-lin.tar.xz +tests: + run_inputs: + - {} + + diff --git a/script/get-intel-sde-tool/run.sh b/script/get-intel-sde-tool/run.sh new file mode 100644 index 000000000..7d4bbb0c5 --- /dev/null +++ b/script/get-intel-sde-tool/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +sde_bin=${MLC_INTEL_SDE_BIN_WITH_PATH} +echo "${sde_bin} --version" + +${sde_bin} --version > tmp-ver.out +test $? -eq 0 || exit $? From 993029a471f780be8a2e02cc02c5e5466a4f165b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 28 Mar 2025 23:00:19 +0000 Subject: [PATCH 13/17] Update meta.yaml (#334) * Update meta.yaml * Fix test script github action --- .github/scripts/list_modified_files.py | 15 +++---- .github/scripts/process_individual_tests.py | 39 ------------------- .github/workflows/build_wheels.yml | 1 + .../workflows/run-individual-script-tests.yml | 37 ------------------ .../workflows/run-tests-on-modified-meta.yml | 32 ++++++++------- .../test-mlc-based-submission-generation.yml | 2 +- script/get-intel-sde-tool/meta.yaml | 1 + 7 files changed, 28 insertions(+), 99 deletions(-) delete mode 100644 .github/scripts/process_individual_tests.py delete mode 100644 .github/workflows/run-individual-script-tests.yml diff --git a/.github/scripts/list_modified_files.py b/.github/scripts/list_modified_files.py index 3a6d4b90f..c3c2260a7 100644 --- a/.github/scripts/list_modified_files.py +++ b/.github/scripts/list_modified_files.py @@ -1,12 +1,13 @@ import yaml import sys import json +import os -def get_num_runs(filepath): +def get_file_info(filepath): with open(filepath, 'r') as file: content = yaml.safe_load(file) - tests = content.get('tests', {}) + tests = content.get('tests', []) if tests: num_tests = len(tests.get('run_inputs', [])) else: @@ -21,11 +22,11 @@ def process_files(files): { "file": file, "uid": uid, - "num_tests": num_tests + "num_run": i } - for file in filenames - if file.endswith('meta.yaml') and (uid := get_num_runs(file)) - for _, num_tests in [uid] + for file in filenames if os.path.basename(file) == 'meta.yaml' + for uid, num_tests in [get_file_info(file)] + for i in range(1, num_tests + 1) ] @@ -33,4 +34,4 @@ def process_files(files): changed_files = sys.stdin.read().strip() processed_files = process_files(changed_files) json_processed_files = json.dumps(processed_files) - print(f"::set-output name=modified_files::{json_processed_files}") + print(f"::set-output name=processed_files::{json_processed_files}") diff --git a/.github/scripts/process_individual_tests.py b/.github/scripts/process_individual_tests.py deleted file mode 100644 index 708a80963..000000000 --- a/.github/scripts/process_individual_tests.py +++ /dev/null @@ -1,39 +0,0 @@ -import sys -import os -import mlc -import json -import yaml - -files = sys.argv[1:] - -for file in files: - print(file) - if not os.path.isfile(file) or not "script" in file: - continue - if not file.endswith("meta.json") and not file.endswith("meta.yaml"): - continue - script_path = os.path.dirname(file) - f = open(file) - if file.endswith(".json"): - data = json.load(f) - elif file.endswith(".yaml"): - data = yaml.safe_load(f) - if data.get('uid', '') == '': - continue # not a CM script meta - uid = data['uid'] - - ii = { - 'action': 'test', 'target': 'script', 'item': uid, 'quiet': 'yes', 'out': 'con' - } - if os.environ.get('DOCKER_MLC_REPO', '') != '': - ii['docker_mlc_repo'] = os.environ['DOCKER_MLC_REPO'] - if os.environ.get('DOCKER_MLC_REPO_BRANCH', '') != '': - ii['docker_mlc_repo_branch'] = os.environ['DOCKER_MLC_REPO_BRANCH'] - if os.environ.get('TEST_INPUT_INDEX', '') != '': - ii['test_input_index'] = os.environ['TEST_INPUT_INDEX'] - print(ii) - ret = mlc.access(ii) - if ret['return'] > 0: - raise Exception(r['error']) - ii = {'action': 'rm', 'target': 'cache', 'f': True} - ret = mlc.access(ii) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 86b93ac06..708ae35e4 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -6,6 +6,7 @@ on: workflow_dispatch: {} jobs: + build_wheels: if: github.repository_owner == 'mlcommons' name: Build wheel diff --git a/.github/workflows/run-individual-script-tests.yml b/.github/workflows/run-individual-script-tests.yml deleted file mode 100644 index b9c31990f..000000000 --- a/.github/workflows/run-individual-script-tests.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This workflow will run configured tests for any updated CM scripts -name: Individual Automation script Tests - -on: - pull_request: - branches: [ "main", "dev" ] - paths: - - 'script/**meta.json' - - 'script/**meta.yaml' - -jobs: - run-script-tests: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test-input-index: [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11+" ] - steps: - - name: 'Checkout' - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - name: Get changed files - id: getfile - run: | - git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} - git fetch upstream - echo "files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only | xargs)" >> $GITHUB_OUTPUT - - name: RUN Script Tests - run: | - echo ${{ steps.getfile.outputs.files }} - for file in ${{ steps.getfile.outputs.files }}; do - echo $file - done - pip install mlcflow - mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - DOCKER_MLC_REPO=${{ github.event.pull_request.head.repo.html_url }} DOCKER_MLC_REPO_BRANCH=${{ github.event.pull_request.head.ref }} TEST_INPUT_INDEX=${{ matrix.test-input-index }} python3 .github/scripts/process_individual_tests.py ${{ steps.getfile.outputs.files }} diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 79ecb1198..7304a42dd 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -1,5 +1,5 @@ # This workflow will run configured tests for any updated MLC script -name: MLC test script on modified meta +name: Test script on modified meta on: pull_request: @@ -8,22 +8,26 @@ on: - 'script/**meta.yaml' jobs: - get-modified-files: + get_modified_files: runs-on: ubuntu-latest outputs: - modified_files: ${{ steps.modified-files.outputs.modified_files }} + processed_files: ${{ steps.modified-files.outputs.processed_files }} steps: - name: 'Checkout' uses: actions/checkout@v4 with: fetch-depth: 2 - + - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: '3.x' - + + - name: Install dependencies + run: | + pip install pyyaml + - name: Get changed files id: modified-files run: | @@ -32,26 +36,24 @@ jobs: changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only) echo "$changed_files" | python3 .github/scripts/list_modified_files.py - - process_modified_files: runs-on: ubuntu-latest needs: get_modified_files strategy: - fail_fast: false + fail-fast: false matrix: - file_info: ${{ fromJSON(needs.get_modified_files.outputs.modified_files) }} - num_run: ${{ range(1, fromJSON(matrix.file_info.num_tests) + 1) }} - + file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }} + steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 2 - name: Process meta.yaml file run: | - echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.num_run }}" + echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}" pip install mlcflow mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet - + mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet diff --git a/.github/workflows/test-mlc-based-submission-generation.yml b/.github/workflows/test-mlc-based-submission-generation.yml index 5c97a2d3e..757e90510 100644 --- a/.github/workflows/test-mlc-based-submission-generation.yml +++ b/.github/workflows/test-mlc-based-submission-generation.yml @@ -5,7 +5,7 @@ on: branches: [ "main", "dev" ] paths: - '.github/workflows/test-mlc-based-submission-generation.yml' - - '**' + - 'script/generate-mlperf-inference-submission/**' - '!**.md' jobs: submission_generation: diff --git a/script/get-intel-sde-tool/meta.yaml b/script/get-intel-sde-tool/meta.yaml index 763593965..60b7de51e 100644 --- a/script/get-intel-sde-tool/meta.yaml +++ b/script/get-intel-sde-tool/meta.yaml @@ -17,6 +17,7 @@ deps: - MLC_DOWNLOAD_URL extra_cache_tags: intel,sde,tool force_cache: True + new_env_keys: - MLC_INTEL_SDE_DIR_PATH - MLC_INTEL_SDE_BIN_WITH_PATH From 3d499ca30b26b3227ec3397bf32b03a58fb6cd32 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 28 Mar 2025 23:09:47 +0000 Subject: [PATCH 14/17] Added tests to the generic script template (#335) --- script/template-script/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/template-script/meta.yaml b/script/template-script/meta.yaml index e6e022b03..f2ed92cd1 100644 --- a/script/template-script/meta.yaml +++ b/script/template-script/meta.yaml @@ -22,3 +22,5 @@ tags: - template uid: fc3e6915a29342d8 +tests: + run_inputs: [] From 4e9fb2ccae156ab067e203f7dff330698d06048b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 29 Mar 2025 02:26:25 +0000 Subject: [PATCH 15/17] Support flang in llvm-install (#336) * Support flang in llvm-install * Fix flang in llvm-install --- script/install-llvm-src/meta.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 0c8d91c43..d788b94dd 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -65,6 +65,13 @@ variations: +MLC_LLVM_ENABLE_PROJECTS: - clang group: clang + flang: + env: + +MLC_LLVM_ENABLE_PROJECTS: + - flang + +MLC_LLVM_ENABLE_RUNTIMES: + - flang-rt + group: flang lld: default: true env: @@ -222,4 +229,6 @@ variations: versions: {} tests: run_inputs: - - version: "20.1.0" + - version: "20.1.1" + - variations_list: + - flang,branch.main From 62572ceb01530c0da95d1c53d7f6dad78f8ce707 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 29 Mar 2025 23:38:57 +0530 Subject: [PATCH 16/17] Add option to skip the certificate check for rclone downloads (#331) * support no check cert for rclone * update customize.py --- script/download-file/customize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 8edbf9413..1081c31e6 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -238,6 +238,8 @@ def preprocess(i): env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer" else: env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer" + if not verify_ssl: + env['MLC_DOWNLOAD_CMD'] += f" --no-check-certificate" filename = env['MLC_DOWNLOAD_FILENAME'] env['MLC_DOWNLOAD_DOWNLOADED_FILENAME'] = filename From 1d794d8fb58eca31070dffc1c4bb60c89f0fd4aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 29 Mar 2025 18:59:13 +0000 Subject: [PATCH 17/17] Fixed meta for get-aocc (#337) --- script/get-aocc/meta.yaml | 2 +- script/install-llvm-src/meta.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index 22adb675c..cead1f6e2 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -77,7 +77,7 @@ variations: env: MLC_AOCC_DOWNLOAD: no path.#: - default_variation: + default_variations: install-type: local-install env: MLC_AOCC_DIR_PATH: '#' diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index d788b94dd..98456233a 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -230,5 +230,5 @@ versions: {} tests: run_inputs: - version: "20.1.1" - - variations_list: - - flang,branch.main +# - variations_list: +# - flang,branch.main