Modularize dmidecode meminfo parsing

.github/workflows/build_wheel.yml

@@ -8,7 +8,6 @@ on:
       - VERSION
 
 jobs:
-
   build_wheels:
     if: github.repository_owner == 'mlcommons'
     name: Build wheel

automation/script/module.py

@@ -1860,6 +1860,7 @@ def _run(self, i):
 
                 run_script_input['meta'] = meta
                 run_script_input['env'] = env
+                run_script_input['state'] = state
                 run_script_input['run_state'] = run_state
                 run_script_input['recursion'] = recursion
 

script/get-mlperf-inference-sut-description/customize.py

@@ -2,6 +2,7 @@
 import os
 import json
 import shutil
+from utils import *
 
 
 def preprocess(i):
@@ -41,6 +42,8 @@ def preprocess(i):
     sut_desc_path = env['MLC_MLPERF_INFERENCE_SUT_DESC_PATH']
 
     sut_path = os.path.join(sut_desc_path, "suts", sut + ".json")
+    env['MLC_SUT_PATH'] = sut_path
+
     if os.path.exists(sut_path) and env.get('MLC_SUT_DESC_CACHE', '') == "yes":
         print(f"Reusing SUT description file {sut}")
         state['MLC_SUT_META'] = json.load(open(sut_path))
@@ -168,24 +171,32 @@ def preprocess(i):
 
         if env.get('MLC_SUDO_USER', '') == "yes" and env.get(
                 'MLC_HOST_OS_TYPE', 'linux'):
+            env['MLC_MEMINFO_FILE'] = os.path.join(os.getcwd(), 'meminfo.dump')
             r = i['automation'].run_native_script(
                 {'run_script_input': i['run_script_input'], 'env': env, 'script_name': 'detect_memory'})
             if r['return'] > 0:
                 return r
-            if env.get('MLC_HOST_MEM_INFO', '') != '':
-                state['MLC_SUT_META']['host_memory_configuration'] = env['MLC_HOST_MEM_INFO']
-
-        state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items()))
-
-        sut_file = open(sut_path, "w")
-        json.dump(state['MLC_SUT_META'], sut_file, indent=4)
-        sut_file.close()
 
     return {'return': 0}
 
 
 def postprocess(i):
 
     env = i['env']
+    state = i['state']
+
+    if env.get('MLC_MEMINFO_OUTFILE', '') != '' and os.path.exists(
+            env['MLC_MEMINFO_OUTFILE']):
+        with open(env['MLC_MEMINFO_OUTFILE'], "r") as f:
+            data = f.read()
+        state['MLC_SUT_META']['host_memory_configuration'] = data
+
+    state['MLC_SUT_META'] = dict(sorted(state['MLC_SUT_META'].items()))
+
+    sut_path = env['MLC_SUT_PATH']
+
+    sut_file = open(sut_path, "w")
+    json.dump(state['MLC_SUT_META'], sut_file, indent=4)
+    sut_file.close()
 
     return {'return': 0}

script/get-mlperf-inference-sut-description/detect_memory.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 if [[ ${MLC_SUDO_USER} == "yes" ]]; then
-  ${MLC_SUDO} dmidecode -t memory > meminfo.out
-  ${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/get_memory_info.py
+  ${MLC_SUDO} dmidecode -t memory > ${MLC_MEMINFO_FILE}
 fi
 test $? -eq 0 || return $?

script/get-mlperf-inference-sut-description/meta.yaml

@@ -29,16 +29,22 @@ deps:
     MLC_HOST_OS_TYPE:
     - linux
   tags: detect,sudo
-- tags: get,generic-python-lib,_package.dmiparser
 - env:
     MLC_CACHE_DIR_ENV_NAME: MLC_MLPERF_INFERENCE_SUT_DESC_PATH
   extra_cache_tags: mlperf,inference,sut,descriptions
   tags: get,cache,dir,_name.mlperf-inference-sut-descriptions
+
+posthook_deps:
+  - tags: parse,dmidecode,memory,info
+    enable_if_env:
+      MLC_MEMINFO_FILE:
+        - on
 docker:
   run: false
 input_mapping:
   name: MLC_HW_NAME
   submitter: MLC_MLPERF_SUBMITTER
+  memory: MLC_DETERMINE_MEMORY_CONFIGURATION
 new_env_keys:
 - MLC_HW_*
 - MLC_SUT_*

script/parse-dmidecode-memory-info/customize.py

@@ -0,0 +1,37 @@
+from mlc import utils
+import os
+import subprocess
+
+
+def preprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    input_file = env.get('MLC_MEMINFO_FILE', '')
+
+    if input_file == '':
+        return {'return': 1, 'error': 'Please provide a valid input file containing the meminfo output from dmidecode -t memory'}
+
+    output_file = env.get(
+        'MLC_MEMINFO_OUTFILE',
+        os.path.join(
+            os.getcwd(),
+            'meminfo.txt'))
+
+    env['MLC_MEMINFO_OUTFILE'] = output_file
+
+    env['MLC_RUN_CMD'] = f"""{env['MLC_PYTHON_BIN_WITH_PATH']} {os.path.join(env['MLC_TMP_CURRENT_SCRIPT_PATH'], "get_memory_info.py")} {input_file} {output_file}"""
+    return {'return': 0}
+
+
+def postprocess(i):
+
+    env = i['env']
+    state = i['state']
+
+    os_info = i['os_info']
+
+    return {'return': 0}

script/get-mlperf-inference-sut-description/get_memory_info.py → script/parse-dmidecode-memory-info/get_memory_info.py

@@ -1,8 +1,20 @@
 import os
 import json
+import argparse
 from dmiparser import DmiParser
 
-with open("meminfo.out", "r") as f:
+# Adding argument parser
+parser = argparse.ArgumentParser(
+    description='Process meminfo and output MLC_HOST_MEM_INFO.')
+parser.add_argument(
+    "input_file",
+    help="Path to the input file (e.g. meminfo.out)")
+parser.add_argument(
+    "output_file",
+    help="Path to the output file (e.g. tmp-run-env.out)")
+args = parser.parse_args()
+
+with open(args.input_file, "r") as f:
     text = f.read()
     parser = DmiParser(text, sort_keys=True, indent=4)
 
@@ -57,5 +69,5 @@
         meminfo.append("; ".join(item['info']))
 
     meminfo_string = ",   ".join(meminfo)
-    with open("tmp-run-env.out", "w") as f:
-        f.write(f"MLC_HOST_MEM_INFO={meminfo_string}")
+    with open(args.output_file, "w") as f:
+        f.write(meminfo_string)

script/parse-dmidecode-memory-info/meta.yaml

@@ -0,0 +1,21 @@
+alias: parse-dmidecode-memory-info
+automation_alias: script
+automation_uid: 5b4e0237da074764
+category: MLC Utils
+deps:
+  - tags: get,generic-python-lib,_package.dmiparser
+new_env_keys:
+  - MLC_MEMINFO_OUTFILE
+input_mapping:
+  input: MLC_MEMINFO_FILE
+  output: MLC_MEMINFO_OUTFILE
+new_state_keys: []
+post_deps: []
+posthook_deps: []
+prehook_deps: []
+tags:
+- dmidecode
+- parse
+- memory
+- info
+uid: 1a6be59b1e564fbf

script/parse-dmidecode-memory-info/run.bat

@@ -0,0 +1,23 @@
+@echo off
+setlocal enabledelayedexpansion
+
+:: Function to exit if the last command failed
+:exit_if_error
+if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
+exit /b 0
+
+:: Function to run a command
+:run
+echo Running:
+echo %1
+echo.
+
+if /I "%MLC_FAKE_RUN%" NEQ "yes" (
+    call %1
+    call :exit_if_error
+)
+exit /b 0
+
+:: Add your run commands here...
+call :run "%MLC_RUN_CMD%"
+

script/parse-dmidecode-memory-info/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+function exit_if_error() {
+  test $? -eq 0 || exit $?
+}
+
+function run() {
+  echo "Running: "
+  echo "$1"
+  echo ""
+  if [[ ${MLC_FAKE_RUN} != 'yes' ]]; then
+    eval "$1"
+    exit_if_error
+  fi
+}
+
+#Add your run commands here...
+run "$MLC_RUN_CMD"