Sync Dev

.github/scripts/list_modified_files.py

@@ -0,0 +1,37 @@
+import yaml
+import sys
+import json
+import os
+
+
+def get_file_info(filepath):
+    with open(filepath, 'r') as file:
+        content = yaml.safe_load(file)
+        tests = content.get('tests', [])
+        if tests:
+            num_tests = len(tests.get('run_inputs', []))
+        else:
+            num_tests = 0
+        uid = content['uid']
+        return uid, num_tests
+
+
+def process_files(files):
+    filenames = files.split()
+    return [
+        {
+            "file": file,
+            "uid": uid,
+            "num_run": i
+        }
+        for file in filenames if os.path.basename(file) == 'meta.yaml'
+        for uid, num_tests in [get_file_info(file)]
+        for i in range(1, num_tests + 1)
+    ]
+
+
+if __name__ == "__main__":
+    changed_files = sys.stdin.read().strip()
+    processed_files = process_files(changed_files)
+    json_processed_files = json.dumps(processed_files)
+    print(f"::set-output name=processed_files::{json_processed_files}")

.github/workflows/build_wheel_off.yml

@@ -8,7 +8,6 @@ on:
       - VERSION
 
 jobs:
-
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

.github/workflows/run-tests-on-modified-meta.yml

@@ -0,0 +1,59 @@
+# This workflow will run configured tests for any updated MLC script
+name: Test script on modified meta
+
+on:
+  pull_request:
+    branches: [ "main", "dev" ]
+    paths:
+      - 'script/**meta.yaml'
+
+jobs:
+  get_modified_files:
+    runs-on: ubuntu-latest
+    outputs:
+      processed_files: ${{ steps.modified-files.outputs.processed_files }}
+
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+
+    - name: Install dependencies
+      run: |
+        pip install pyyaml
+
+    - name: Get changed files
+      id: modified-files
+      run: |
+        git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }}
+        git fetch upstream
+        changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only)
+        echo "$changed_files" | python3 .github/scripts/list_modified_files.py
+
+  process_modified_files:
+    runs-on: ubuntu-latest
+    needs: get_modified_files
+    strategy:
+      fail-fast: false
+      matrix:
+        file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Process meta.yaml file
+        run: |
+          echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}"
+
+          pip install mlcflow
+          mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+          mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet

.github/workflows/test-mlc-based-submission-generation.yml

@@ -5,7 +5,7 @@ on:
     branches: [ "main", "dev" ]
     paths:
       - '.github/workflows/test-mlc-based-submission-generation.yml'
-      - '**'  
+      - 'script/generate-mlperf-inference-submission/**'  
       - '!**.md'
 jobs:
   submission_generation:

.github/workflows/test-mlc-script-pip.yml

@@ -0,0 +1,59 @@
+name: Test pip installation for mlc-script
+
+on:
+  pull_request:
+    branches: [ "main", "dev" ]
+    paths:
+      - '.github/workflows/test-mlc-script-pip.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  test_mlc_script_install_pypi:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.8"]
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        exclude:
+          - os: windows-latest
+          - os: macos-latest
+
+    steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: install mlc-scripts
+      run: |
+        pip install mlc-scripts
+    - name: run detect-os script
+      run: |
+        mlcr detect-os -j
+
+  test_mlc_script_install_pull_request_source:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.8"]
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        exclude:
+          - os: windows-latest
+          - os: macos-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: install mlc-scripts from source repo
+      run: |
+        pip install .
+    - name: run detect-os script
+      run: |
+        mlcr detect-os -j

automation/script/module.py

@@ -2929,7 +2929,7 @@ def test(self, i):
 
             alias = meta.get('alias', '')
             uid = meta.get('uid', '')
-            if console:
+            if console or True:  # Todo restrict to console only?
                 logger.info(path)
                 test_config = meta.get('tests', '')
                 if test_config:

script/download-and-extract/meta.yaml

@@ -19,6 +19,7 @@ input_mapping:
 new_env_keys:
 - MLC_DOWNLOAD_DOWNLOADED_PATH*
 - MLC_EXTRACT_EXTRACTED_PATH
+- MLC_EXTRACT_EXTRACTED_SUBDIR_PATH
 - <<<MLC_DOWNLOAD_FINAL_ENV_NAME>>>
 - <<<MLC_EXTRACT_FINAL_ENV_NAME>>>
 - <<<MLC_DAE_FINAL_ENV_NAME>>>

script/download-file/customize.py

@@ -76,7 +76,7 @@ def preprocess(i):
         print('')
         print('Downloading from {}'.format(url))
 
-        if '&' in url and tool != "cmutil":
+        if '&' in url and tool != "mlcutil":
             if os_info['platform'] == 'windows':
                 url = '"' + url + '"'
             else:
@@ -112,8 +112,8 @@ def preprocess(i):
             else:
                 env['MLC_DOWNLOAD_FILENAME'] = "index.html"
 
-        if tool == "cmutil":
-            cmutil_require_download = 0
+        if tool == "mlcutil":
+            mlcutil_require_download = 0
             if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '':
                 if os_info['platform'] == 'windows':
                     checksum_cmd = f"cd {q}{filepath}{q} {xsep}  md5sum -c{x_c} {x}{escape_special_chars(env['MLC_DOWNLOAD_CHECKSUM_FILE'])}"
@@ -126,7 +126,7 @@ def preprocess(i):
                     text=True,
                     shell=True,
                     env=subprocess_env)
-            elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '':
+            elif env.get('MLC_DOWNLOAD_CHECKSUM', '') != '' and os.path.isfile(env['MLC_DOWNLOAD_FILENAME']):
                 if os_info['platform'] == 'windows':
                     checksum_cmd = f"echo {env.get('MLC_DOWNLOAD_CHECKSUM')} {x}{escape_special_chars(env['MLC_DOWNLOAD_FILENAME'])} | md5sum -c{x_c} -"
                 else:
@@ -137,8 +137,8 @@ def preprocess(i):
                     text=True,
                     shell=True,
                     env=subprocess_env)
-            if env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get(
-                    'MLC_DOWNLOAD_CHECKSUM', '') != '':
+            if (env.get('MLC_DOWNLOAD_CHECKSUM_FILE', '') != '' or env.get(
+                    'MLC_DOWNLOAD_CHECKSUM', '') != '') and os.path.exists(env['MLC_DOWNLOAD_FILENAME']):
                 # print(checksum_result) #for debugging
                 if "checksum did not match" in checksum_result.stderr.lower():
                     computed_checksum = subprocess.run(
@@ -155,21 +155,20 @@ def preprocess(i):
                     except PermissionError:
                         return {
                             "return": 1, "error": f"Permission denied to delete file {env['MLC_DOWNLOAD_FILENAME']}."}
-                    cmutil_require_download = 1
+                    mlcutil_require_download = 1
                 elif "no such file" in checksum_result.stderr.lower():
-                    # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. Downloading through cmutil.")
-                    cmutil_require_download = 1
+                    # print(f"No file {env['MLC_DOWNLOAD_FILENAME']}. Downloading through mlcutil.")
+                    mlcutil_require_download = 1
                 elif checksum_result.returncode > 0:
                     return {
                         "return": 1, "error": f"Error while checking checksum: {checksum_result.stderr}"}
                 else:
                     print(
                         f"File {env['MLC_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..")
             else:
-                cmutil_require_download = 1
+                mlcutil_require_download = 1
 
-            if cmutil_require_download == 1:
-                cm = automation.action_object
+            if mlcutil_require_download == 1:
                 for i in range(1, 5):
                     r = download_file({
                         'url': url,
@@ -239,6 +238,8 @@ def preprocess(i):
                 env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer"
             else:
                 env['MLC_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['MLC_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer"
+            if not verify_ssl:
+                env['MLC_DOWNLOAD_CMD'] += f" --no-check-certificate"
 
         filename = env['MLC_DOWNLOAD_FILENAME']
         env['MLC_DOWNLOAD_DOWNLOADED_FILENAME'] = filename

script/download-file/meta.yaml

@@ -45,7 +45,7 @@ variations:
   mlcutil:
     default: true
     env:
-      MLC_DOWNLOAD_TOOL: cmutil
+      MLC_DOWNLOAD_TOOL: mlcutil
     group: download-tool
   curl:
     default_env:

script/download-file/run.bat

@@ -19,7 +19,7 @@ if not "%MLC_DOWNLOAD_LOCAL_FILE_PATH%" == "" (
   set require_download=0
 )
 
-if "%MLC_DOWNLOAD_TOOL%" == "cmutil" (
+if "%MLC_DOWNLOAD_TOOL%" == "mlcutil" (
   set require_download=0
 )
 

script/download-file/run.sh

@@ -9,8 +9,8 @@ fi
 # Assume download is required by default
 require_download=1
 
-# No download needed if a local file path is specified or the tool is 'cmutil'
-if [[ -n "${MLC_DOWNLOAD_LOCAL_FILE_PATH}" || ${MLC_DOWNLOAD_TOOL} == "cmutil" ]]; then
+# No download needed if a local file path is specified or the tool is 'mlcutil'
+if [[ -n "${MLC_DOWNLOAD_LOCAL_FILE_PATH}" || ${MLC_DOWNLOAD_TOOL} == "mlcutil" ]]; then
   require_download=0
 fi
 
@@ -50,7 +50,7 @@ if [[ ${require_download} == 1 ]]; then
 fi
 
 # Verify checksum again if necessary
-if [[ ${MLC_DOWNLOAD_TOOL} == "cmutil" || ${require_download} == 1 ]]; then
+if [[ ${MLC_DOWNLOAD_TOOL} == "mlcutil" || ${require_download} == 1 ]]; then
   if [[ -n "${MLC_DOWNLOAD_CHECKSUM_CMD}" ]]; then
     echo -e "\nVerifying checksum after download: ${MLC_DOWNLOAD_CHECKSUM_CMD}"
     eval "${MLC_DOWNLOAD_CHECKSUM_CMD}" || exit $?