From cbdf92f453aff7acdf9d00c5fee1c499bbe67bd6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 17 May 2025 19:23:59 +0530 Subject: [PATCH 1/5] Support mlc experiment entries --- automation/script/experiment.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index 5195ce056..4091b7d67 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -1,5 +1,7 @@ from collections import defaultdict import os +from mlc.main import ExperimentAction +import mlc.utils as utils from mlc import utils from utils import * import logging @@ -26,6 +28,8 @@ def experiment_run(self_module, i): show_time = i.get('show_time', False) logger = self_module.logger env = i.get('env', {}) + experiment_action = ExperimentAction(self_module.action_object.parent) + prune_result = prune_input( {'input': i, 'extra_keys_starts_with': ['exp.']}) if prune_result['return'] > 0: @@ -80,6 +84,18 @@ def experiment_run(self_module, i): if r['return'] > 0: return r + experiment_meta = {} + exp_tags = tags + ii = {'action': 'update', + 'target': 'experiment', + 'script_alias': meta['alias'], + 'tags': ','.join(exp_tags), + 'meta': experiment_meta, + 'force': True} + r = experiment_action.access(ii) + if r['return'] > 0: + return r + return {'return': 0} From 5f6fc6a7ca87c8207c754c29e0c9a01585ecd7f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 17 May 2025 13:54:17 +0000 Subject: [PATCH 2/5] [Automated Commit] Format Codebase [skip ci] --- automation/script/experiment.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automation/script/experiment.py b/automation/script/experiment.py index 4091b7d67..63a29f005 100644 --- a/automation/script/experiment.py +++ b/automation/script/experiment.py @@ -87,11 +87,11 @@ def experiment_run(self_module, i): experiment_meta = {} exp_tags = tags ii = {'action': 'update', - 'target': 'experiment', - 'script_alias': meta['alias'], - 'tags': ','.join(exp_tags), - 'meta': experiment_meta, - 'force': True} + 'target': 'experiment', + 'script_alias': meta['alias'], + 'tags': ','.join(exp_tags), + 'meta': experiment_meta, + 'force': True} r = experiment_action.access(ii) if r['return'] > 0: return r From 173bc35bfb88f98a3ab0d76cd488016a8c5d6191 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 17 May 2025 20:16:34 +0530 Subject: [PATCH 3/5] Added save-machine-state script --- script/save-machine-state/capture.py | 99 ++++++++++++++++++++++++++ script/save-machine-state/customize.py | 47 ++++++++++++ script/save-machine-state/meta.yaml | 67 +++++++++++++++++ script/save-machine-state/run.sh | 4 ++ 4 files changed, 217 insertions(+) create mode 100644 script/save-machine-state/capture.py create mode 100644 script/save-machine-state/customize.py create mode 100644 script/save-machine-state/meta.yaml create mode 100644 script/save-machine-state/run.sh diff --git a/script/save-machine-state/capture.py b/script/save-machine-state/capture.py new file mode 100644 index 000000000..ca46cc76a --- /dev/null +++ b/script/save-machine-state/capture.py @@ -0,0 +1,99 @@ +import os +import json +import psutil +import platform +import subprocess +from datetime import datetime + +def read_file_safe(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except Exception: + return None + +def run_command_safe(command, require_sudo=False): + if require_sudo and os.geteuid() != 0: + return "Skipped (requires sudo)" + try: + output = subprocess.check_output(command, shell=True, text=True) + return output.strip() + except subprocess.CalledProcessError: + return "Error running command" + +def detect_container_context(): + context = { + "docker_env": os.path.exists('/.dockerenv'), + "cgroup_indicators": [] + } + cgroup = read_file_safe('/proc/1/cgroup') + if cgroup: + for line in cgroup.splitlines(): + if any(x in line for x in ['docker', 'kubepods', 'containerd']): + context["cgroup_indicators"].append(line) + return context + +def get_mounted_file_systems(): + try: + with open("/proc/mounts", "r") as f: + return [line.strip() for line in f.readlines()] + except: + return [] + +def capture_machine_state(): + state = { + "timestamp": datetime.now().isoformat(), + "platform": { + "system": platform.system(), + "node": platform.node(), + "release": platform.release(), + "version": platform.version(), + "machine": platform.machine(), + "processor": platform.processor() + }, + "cpu": { + "logical_cores": psutil.cpu_count(logical=True), + "physical_cores": psutil.cpu_count(logical=False), + "load_avg": psutil.getloadavg(), + "cpu_percent": psutil.cpu_percent(interval=1) + }, + "memory": { + "virtual_memory": dict(psutil.virtual_memory()._asdict()), + "swap_memory": dict(psutil.swap_memory()._asdict()) + }, + "disk": { + "disk_usage": dict(psutil.disk_usage('/')._asdict()), + "partitions": [dict(p._asdict()) for p in psutil.disk_partitions()] + }, + "bios": { + "vendor": run_command_safe("dmidecode -s bios-vendor", require_sudo=True), + "version": run_command_safe("dmidecode -s bios-version", require_sudo=True), + "release_date": run_command_safe("dmidecode -s bios-release-date", require_sudo=True) + }, + "thp_settings": { + "enabled": read_file_safe("/sys/kernel/mm/transparent_hugepage/enabled") or "Skipped (requires sudo or permission)", + "defrag": read_file_safe("/sys/kernel/mm/transparent_hugepage/defrag") or "Skipped (requires sudo or permission)" + }, + "kernel": { + "cmdline": read_file_safe("/proc/cmdline") + }, + "uptime": read_file_safe("/proc/uptime"), + "process_count": len(psutil.pids()), + "users_sessions": [dict(u._asdict()) for u in psutil.users()], + "container_context": detect_container_context(), + "mounted_filesystems": get_mounted_file_systems() + } + return state + +def save_state_to_file(state, filename): + with open(filename, "w") as f: + json.dump(state, f, indent=4) + + +# Example usage +if __name__ == "__main__": + + state = capture_machine_state() + save_file = os.environ.get('MLC_SYSTEM_STATE_SAVE_FILENAME', 'machine_state.json') + save_state_to_file(state, save_file) + diff --git a/script/save-machine-state/customize.py b/script/save-machine-state/customize.py new file mode 100644 index 000000000..a7dbcc4f2 --- /dev/null +++ b/script/save-machine-state/customize.py @@ -0,0 +1,47 @@ +from mlc import utils +import os +import subprocess + + +def check_installation(command, os_info): + if os_info['platform'] == "windows": + return subprocess.call( + [command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 + elif os_info['platform'] == "linux": + return subprocess.call(['which', command], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) == 0 # 0 means the package is there + + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + + if not check_installation("numactl", os_info): + env['MLC_INSTALL_NUMACTL'] = 'True' + + # if not check_installation("cpupower",os_info): + env['MLC_INSTALL_CPUPOWER'] = 'True' + + if env.get('MLC_PLATFORM_DETAILS_FILE_PATH', '') == '': + if env.get('MLC_PLATFORM_DETAILS_DIR_PATH', '') == '': + env['MLC_PLATFORM_DETAILS_DIR_PATH'] = os.getcwd() + if env.get('MLC_PLATFORM_DETAILS_FILE_NAME', '') == '': + env['MLC_PLATFORM_DETAILS_FILE_NAME'] = "system-info.txt" + env['MLC_PLATFORM_DETAILS_FILE_PATH'] = os.path.join( + env['MLC_PLATFORM_DETAILS_DIR_PATH'], env['MLC_PLATFORM_DETAILS_FILE_NAME']) + + return {'return': 0} + + +def postprocess(i): + + state = i['state'] + + env = i['env'] + + os_info = i['os_info'] + + automation = i['automation'] + + return {'return': 0} diff --git a/script/save-machine-state/meta.yaml b/script/save-machine-state/meta.yaml new file mode 100644 index 000000000..7d9a07cd1 --- /dev/null +++ b/script/save-machine-state/meta.yaml @@ -0,0 +1,67 @@ +alias: save-machine-state +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: false +category: Platform information +deps: +- tags: detect,os +- skip_if_env: + MLC_HOST_OS_TYPE: + - windows + tags: detect,sudo +- names: + - python + - python3 + tags: get,python +- skip_if_any_env: + MLC_HOST_OS_TYPE: + - windows + skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_psmisc +- enable_if_env: + MLC_HOST_OS_TYPE: + - linux + skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_systemd +- enable_if_env: + MLC_HOST_OS_TYPE: + - linux + skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_dmidecode +- tags: get,generic-python-lib,_package.psutil +input_mapping: + outfile: MLC_SYSTEM_STATE_SAVE_FILENAME +prehook_deps: +- enable_if_env: + MLC_HOST_OS_TYPE: + - linux + MLC_INSTALL_NUMACTL: + - 'True' + skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_numactl +- enable_if_env: + MLC_HOST_OS_TYPE: + - linux + MLC_INSTALL_CPUPOWER: + - 'True' + env: + MLC_TMP_FAIL_SAFE: 'yes' + ignore_missing: true + skip_if_env: + MLC_SUDO_USER: + - 'no' + tags: get,sys-util,generic,_linux-tools +tags: +- machine-state +- save +- machine +- state +uid: 2f62820ed7294659 diff --git a/script/save-machine-state/run.sh b/script/save-machine-state/run.sh new file mode 100644 index 000000000..28e1867f6 --- /dev/null +++ b/script/save-machine-state/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +${MLC_PYTHON_BIN_WITH_PATH} ${MLC_TMP_CURRENT_SCRIPT_PATH}/capture.py From cd21cfb1537382c424fb4bce32ed7188093a9110 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 17 May 2025 14:46:55 +0000 Subject: [PATCH 4/5] [Automated Commit] Format Codebase [skip ci] --- script/save-machine-state/capture.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/script/save-machine-state/capture.py b/script/save-machine-state/capture.py index ca46cc76a..02963eb81 100644 --- a/script/save-machine-state/capture.py +++ b/script/save-machine-state/capture.py @@ -5,6 +5,7 @@ import subprocess from datetime import datetime + def read_file_safe(path): try: with open(path, 'r') as f: @@ -12,6 +13,7 @@ def read_file_safe(path): except Exception: return None + def run_command_safe(command, require_sudo=False): if require_sudo and os.geteuid() != 0: return "Skipped (requires sudo)" @@ -21,6 +23,7 @@ def run_command_safe(command, require_sudo=False): except subprocess.CalledProcessError: return "Error running command" + def detect_container_context(): context = { "docker_env": os.path.exists('/.dockerenv'), @@ -33,13 +36,15 @@ def detect_container_context(): context["cgroup_indicators"].append(line) return context + def get_mounted_file_systems(): try: with open("/proc/mounts", "r") as f: return [line.strip() for line in f.readlines()] - except: + except BaseException: return [] + def capture_machine_state(): state = { "timestamp": datetime.now().isoformat(), @@ -85,6 +90,7 @@ def capture_machine_state(): } return state + def save_state_to_file(state, filename): with open(filename, "w") as f: json.dump(state, f, indent=4) @@ -92,8 +98,9 @@ def save_state_to_file(state, filename): # Example usage if __name__ == "__main__": - + state = capture_machine_state() - save_file = os.environ.get('MLC_SYSTEM_STATE_SAVE_FILENAME', 'machine_state.json') + save_file = os.environ.get( + 'MLC_SYSTEM_STATE_SAVE_FILENAME', + 'machine_state.json') save_state_to_file(state, save_file) - From a767ff8ea4cf630329395afa9953fd6e188c9f4b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 17 May 2025 15:50:21 +0100 Subject: [PATCH 5/5] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 86b93ac06..55e79c0ae 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,6 +5,7 @@ on: types: [published] workflow_dispatch: {} + jobs: build_wheels: if: github.repository_owner == 'mlcommons'