Sync Dev

.github/workflows/build_wheel.yml → .github/workflows/build_wheel_off.yml

@@ -1,4 +1,4 @@
-name: Build wheel and release into PYPI
+name: Build wheel and release into PYPI (off now)
 
 on: 
   push:

.github/workflows/build_wheels_on_release.yml → .github/workflows/build_wheels.yml

@@ -1,4 +1,4 @@
-name: Build wheel and release into PYPI
+name: Build wheel (only src) and release into PYPI
 
 on:
   release:

VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

automation/script/module.py

@@ -1860,6 +1860,7 @@ def _run(self, i):
 
                 run_script_input['meta'] = meta
                 run_script_input['env'] = env
+                run_script_input['state'] = state
                 run_script_input['run_state'] = run_state
                 run_script_input['recursion'] = recursion
 
@@ -4726,6 +4727,9 @@ def find_cached_script(i):
                     r = docker_utils.get_container_path_script(i)
                     if not os.path.exists(r['value_env']):
                         # Need to rm this cache entry
+                        logger.debug(
+                            recursion_spaces +
+                            '  - Skipping cached entry as the dependent path {} is missing!'.format(r['value_env']))
                         skip_cached_script = True
                         continue
 

script/download-file/meta.yaml

@@ -6,6 +6,7 @@ can_force_cache: true
 category: DevOps automation
 default_env:
   MLC_RCLONE_COPY_USING: sync
+  MLC_EXTRACT_REMOVE_EXTRACTED: no
 deps:
 - tags: detect,os
 - enable_if_env:
@@ -40,6 +41,8 @@ tags_help: download file
 uid: 9cdc8dc41aae437e
 variations:
   cmutil:
+    alias: mlcutil
+  mlcutil:
     default: true
     env:
       MLC_DOWNLOAD_TOOL: cmutil

script/extract-file/customize.py

@@ -1,6 +1,7 @@
 from mlc import utils
 import os
 import hashlib
+from utils import *
 
 
 def preprocess(i):
@@ -209,7 +210,7 @@ def postprocess(i):
     env['MLC_GET_DEPENDENT_CACHED_PATH'] = filepath
 
     # Check if need to remove archive after extraction
-    if env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '').lower() != 'no':
+    if is_true(env.get('MLC_EXTRACT_REMOVE_EXTRACTED', '')):
         archive_filepath = env.get('MLC_EXTRACT_FILEPATH', '')
         if archive_filepath != '' and os.path.isfile(archive_filepath):
             os.remove(archive_filepath)

script/extract-file/meta.yaml

@@ -36,9 +36,12 @@ tags_help: extract file
 uid: 3f0b76219d004817
 variations:
   keep:
+    group: keep
+    default: true
     env:
       MLC_EXTRACT_REMOVE_EXTRACTED: 'no'
   no-remove-extracted:
+    group: keep
     env:
       MLC_EXTRACT_REMOVE_EXTRACTED: 'no'
   path.#:

script/get-aocc/customize.py

@@ -25,6 +25,13 @@ def preprocess(i):
             aocc_path = env['MLC_AOCC_DIR_PATH']
             if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')):
                 env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')
+            else:
+                for l in os.listdir(aocc_path):
+                    if os.path.exists(os.path.join(
+                            aocc_path, l, 'bin', 'clang')):
+                        aocc_path = os.path.join(aocc_path, l)
+                        env['MLC_AOCC_DIR_PATH'] = aocc_path
+                        env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin')
 
         r = i['automation'].find_artifact({'file_name': exe_c,
                                            'env': env,
@@ -42,7 +49,7 @@ def preprocess(i):
 
 
 def detect_version(i):
-    r = i['automation'].parse_version({'match_text': r'AMD\s+clang\sversion\s([\d.]+)',
+    r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)',
                                        'group_number': 1,
                                        'env_key': 'MLC_AOCC_VERSION',
                                        'which_env': i['env']})

script/get-aocc/meta.yaml

@@ -6,10 +6,24 @@ category: Compiler automation
 clean_files: []
 deps:
 - tags: detect,os
+- tags: download,file,_mlcutil
+  extra_cache_tags: aocc,download
+  update_tags_from_env_with_prefix:
+    _url.:
+      - MLC_AOCC_URL
+  force_cache: true
+  env:
+    MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_AOCC_TAR_FILE_PATH
+  enable_if_env:
+    MLC_AOCC_URL:
+      - on
+    MLC_AOCC_ACCEPT_EULA:
+      - on
 - tags: extract,file
   update_tags_from_env_with_prefix:
     _path.:
       - MLC_AOCC_TAR_FILE_PATH
+  extra_cache_tags: aocc,extract
   force_cache: true
   env:
     MLC_EXTRACT_FINAL_ENV_NAME: MLC_AOCC_DIR_PATH
@@ -21,6 +35,7 @@ deps:
 input_mapping:
   tar_file_path: MLC_AOCC_TAR_FILE_PATH
   aocc_dir: MLC_AOCC_DIR_PATH
+  accept_eula: MLC_AOCC_ACCEPT_EULA
 
 name: Detect or install AOCC compiler
 new_env_keys:
@@ -43,7 +58,14 @@ tags:
 - get
 - aocc
 uid: 1ceb0656e99a44ec
+versions:
+  5.0.0:
+    env:
+      MLC_AOCC_URL: https://download.amd.com/developer/eula/aocc/aocc-5-0/aocc-compiler-5.0.0.tar
+      MLC_DOWNLOAD_CHECKSUM: 966fac2d2c759e9de6e969c10ada7a7b306c113f7f1e07ea376829ec86380daa
+      MLC_AOCC_NEEDS_TAR: yes
+      MLC_VERSION: '5.0.0-Build#1377' 
 variations:
-  _path.#:
+  path.#:
     env:
-      MLC_AOCC_DIR_PATH: #
+      MLC_AOCC_DIR_PATH: '#'

script/get-gcc/meta.yaml

@@ -31,6 +31,6 @@ tags:
 - get-gcc
 uid: dbf4ab5cbed74372
 variations:
-  _path.#:
+  path.#:
     env:
-      MLC_GCC_DIR_PATH: #
+      MLC_GCC_DIR_PATH: '#'

script/get-mlperf-inference-sut-description/customize.py

@@ -2,6 +2,7 @@
 import os
 import json
 import shutil
+from utils import *
 
 
 def preprocess(i):
@@ -41,6 +42,8 @@ def preprocess(i):
     sut_desc_path = env['MLC_MLPERF_INFERENCE_SUT_DESC_PATH']
 
     sut_path = os.path.join(sut_desc_path, "suts", sut + ".json")
+    env['MLC_SUT_PATH'] = sut_path
+
     if os.path.exists(sut_path) and env.get('MLC_SUT_DESC_CACHE', '') == "yes":
         print(f"Reusing SUT description file {sut}")
         state['MLC_SUT_META'] = json.load(open(sut_path))
@@ -168,24 +171,32 @@ def preprocess(i):
 
         if env.get('MLC_SUDO_USER', '') == "yes" and env.get(
                 'MLC_HOST_OS_TYPE', 'linux'):
+            env['MLC_MEMINFO_FILE'] = os.path.join(os.getcwd(), 'meminfo.dump')
             r = i['automation'].run_native_script(
                 {'run_script_input': i['run_script_input'], 'env': env, 'script_name': 'detect_memory'})
             if r['return'] > 0:
                 return r
-            if env.get('MLC_HOST_MEM_INFO', '') != '':
-                state['MLC_SUT_META']['host_memory_configuration'] = env['MLC_HOST_MEM_INFO']
-
-        state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items()))
-
-        sut_file = open(sut_path, "w")
-        json.dump(state['MLC_SUT_META'], sut_file, indent=4)
-        sut_file.close()
 
     return {'return': 0}
 
 
 def postprocess(i):
 
     env = i['env']
+    state = i['state']
+
+    if env.get('MLC_MEMINFO_OUTFILE', '') != '' and os.path.exists(
+            env['MLC_MEMINFO_OUTFILE']):
+        with open(env['MLC_MEMINFO_OUTFILE'], "r") as f:
+            data = f.read()
+        state['MLC_SUT_META']['host_memory_configuration'] = data
+
+    state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items()))
+
+    sut_path = env['MLC_SUT_PATH']
+
+    sut_file = open(sut_path, "w")
+    json.dump(state['MLC_SUT_META'], sut_file, indent=4)
+    sut_file.close()
 
     return {'return': 0}

script/get-mlperf-inference-sut-description/detect_memory.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
-  ${MLC_SUDO} dmidecode -t memory > meminfo.out
-  ${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/get_memory_info.py
+  ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE}
 fi
 test $? -eq 0 || return $?

script/get-mlperf-inference-sut-description/meta.yaml

@@ -29,16 +29,22 @@ deps:
     MLC_HOST_OS_TYPE:
     - linux
   tags: detect,sudo
-- tags: get,generic-python-lib,_package.dmiparser
 - env:
     MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH
   extra_cache_tags: mlperf,inference,sut,descriptions
   tags: get,cache,dir,_name.mlperf-inference-sut-descriptions
+
+posthook_deps:
+  - tags: parse,dmidecode,memory,info
+    enable_if_env:
+      MLC_MEMINFO_FILE:
+        - on
 docker:
   run: false
 input_mapping:
   name: MLC_HW_NAME
   submitter: MLC_MLPERF_SUBMITTER
+  memory: MLC_DETERMINE_MEMORY_CONFIGURATION
 new_env_keys:
 - MLC_HW_*
 - MLC_SUT_*

script/get-one-api/meta.yaml

@@ -26,6 +26,6 @@ tags:
 - get-oneapi
 uid: 1af872e81ef54742
 variations:
-  _path.#:
+  path.#:
     env:
       MLC_ONEAPI_DIR_PATH: "#"

script/parse-dmidecode-memory-info/customize.py

@@ -0,0 +1,37 @@
+from mlc import utils
+import os
+import subprocess
+
+
+def preprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    input_file = env.get('MLC_MEMINFO_FILE', '')
+
+    if input_file == '':
+        return {'return': 1, 'error': 'Please provide a valid input file containing the meminfo output from dmidecode -t memory'}
+
+    output_file = env.get(
+        'MLC_MEMINFO_OUTFILE',
+        os.path.join(
+            os.getcwd(),
+            'meminfo.txt'))
+
+    env['MLC_MEMINFO_OUTFILE'] = output_file
+
+    env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'], "get_memory_info.py")} {input_file} {output_file}"""
+    return {'return': 0}
+
+
+def postprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    return {'return': 0}

script/get-mlperf-inference-sut-description/get_memory_info.py → script/parse-dmidecode-memory-info/get_memory_info.py

@@ -1,8 +1,20 @@
 import os
 import json
+import argparse
 from dmiparser import DmiParser
 
-with open("meminfo.out", "r") as f:
+# Adding argument parser
+parser = argparse.ArgumentParser(
+    description='Process meminfo and output MLC_HOST_MEM_INFO.')
+parser.add_argument(
+    "input_file",
+    help="Path to the input file (e.g. meminfo.out)")
+parser.add_argument(
+    "output_file",
+    help="Path to the output file (e.g. tmp-run-env.out)")
+args = parser.parse_args()
+
+with open(args.input_file, "r") as f:
     text = f.read()
     parser = DmiParser(text, sort_keys=True, indent=4)
 
@@ -57,5 +69,5 @@
         meminfo.append("; ".join(item['info']))
 
     meminfo_string = ",   ".join(meminfo)
-    with open("tmp-run-env.out", "w") as f:
-        f.write(f"MLC_HOST_MEM_INFO={meminfo_string}")
+    with open(args.output_file, "w") as f:
+        f.write(meminfo_string)

script/parse-dmidecode-memory-info/meta.yaml

@@ -0,0 +1,21 @@
+alias: parse-dmidecode-memory-info
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: MLC Utils
+deps:
+  - tags: get,generic-python-lib,_package.dmiparser
+new_env_keys:
+  - MLC_MEMINFO_OUTFILE
+input_mapping:
+  input: MLC_MEMINFO_FILE
+  output: MLC_MEMINFO_OUTFILE
+new_state_keys: []
+post_deps: []
+posthook_deps: []
+prehook_deps: []
+tags:
+- dmidecode
+- parse
+- memory
+- info
+uid: 1a6be59b1e564fbf

script/parse-dmidecode-memory-info/run.bat

@@ -0,0 +1,23 @@
+@echo off
+setlocal enabledelayedexpansion
+
+:: Function to exit if the last command failed
+:exit_if_error
+if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
+exit /b 0
+
+:: Function to run a command
+:run
+echo Running:
+echo %1
+echo.
+
+if /I "%MLC_FAKE_RUN%" NEQ "yes" (
+    call %1
+    call :exit_if_error
+)
+exit /b 0
+
+:: Add your run commands here...
+call :run "%MLC_RUN_CMD%"
+

script/parse-dmidecode-memory-info/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+function exit_if_error() {
+  test $? -eq 0 || exit $?
+}
+
+function run() {
+  echo "Running: "
+  echo "$1"
+  echo ""
+  if [[ ${MLC_FAKE_RUN} != 'yes' ]]; then
+    eval "$1"
+    exit_if_error
+  fi
+}
+
+#Add your run commands here...
+run "$MLC_RUN_CMD"