From 77220d5c89714fc8b07d23792147b84580ecafef Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:11:19 +0530 Subject: [PATCH 01/27] updated mixtral dataset download based on latest inference readme (#284) --- .../meta.yaml | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/script/get-dataset-mlperf-inference-mixtral/meta.yaml b/script/get-dataset-mlperf-inference-mixtral/meta.yaml index 99b8e0c2f..744b5690d 100644 --- a/script/get-dataset-mlperf-inference-mixtral/meta.yaml +++ b/script/get-dataset-mlperf-inference-mixtral/meta.yaml @@ -16,14 +16,48 @@ prehook_deps: update_tags_from_env_with_prefix: _url.: - MLC_PACKAGE_URL + names: + - download-file tags: - get - dataset-mixtral - openorca-mbxp-gsm8k-combined uid: 89e7c91444804775 variations: - mlcommons-storage: + rclone: + group: download-tool + default: true + adr: + download-file: + tags: _rclone + env: + MLC_DOWNLOAD_TOOL: rclone + MLC_RCLONE_COPY_USING: copyurl + wget: + group: download-tool + adr: + download-file: + tags: _wget + env: + MLC_DOWNLOAD_TOOL: wget + validation: default: true + group: dataset-type + adr: + download-file: + extra_cache_tags: mixtral,get-mixtral-dataset,validation + env: + ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#preprocessed + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b/09292024_mixtral_15k_mintoken2_v1.pkl + calibration: + group: dataset-type + adr: + download-file: + extra_cache_tags: mixtral,get-mixtral-dataset,calibration + env: + ML_MODEL_STARTING_WEIGHTS_FILENAME: https://github.com/mlcommons/inference/tree/master/language/mixtral-8x7b#calibration-dataset + MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/mixtral_8x7b%2F2024.06.06_mixtral_15k_calibration_v4.pkl + mlcommons-storage: env: MLC_DOWNLOAD_CHECKSUM: 78823c13e0e73e518872105c4b09628b MLC_DOWNLOAD_FILENAME: 2024.06.06_mixtral_15k_v4.pkl From 7a5f1e1822ec757c84a07240fa7b36b1f6bf9889 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 4 Mar 2025 04:45:22 +0530 Subject: [PATCH 02/27] updated download url - llama3 (#285) --- script/get-dataset-mlperf-inference-llama3/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 1028ab0cf..9e8762f35 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -33,13 +33,13 @@ variations: default: true group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl calibration: group: dataset-type env: - MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl rclone: From cfa5519c83b3589d2c9071501c58040ed0a639e9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 4 Mar 2025 14:13:34 +0530 Subject: [PATCH 03/27] Fix argument issue in coco2014 calibration dataset download (#286) * add num of workers * Update run.sh --- script/get-dataset-coco2014/meta.yaml | 6 +++++- script/get-dataset-coco2014/run.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-coco2014/meta.yaml b/script/get-dataset-coco2014/meta.yaml index fa3724f83..afdff9cc4 100644 --- a/script/get-dataset-coco2014/meta.yaml +++ b/script/get-dataset-coco2014/meta.yaml @@ -9,6 +9,9 @@ cache: true category: AI/ML datasets category_sort: 8500 +input_mapping: + num_workers: MLC_DATASET_COCO2014_NUM_WORKERS + tags: - get - dataset @@ -18,7 +21,8 @@ tags: default_env: MLC_DATASET_CALIBRATION: 'no' - + MLC_DATASET_COCO2014_NUM_WORKERS: 1 + deps: - names: diff --git a/script/get-dataset-coco2014/run.sh b/script/get-dataset-coco2014/run.sh index a891b2330..21829568e 100644 --- a/script/get-dataset-coco2014/run.sh +++ b/script/get-dataset-coco2014/run.sh @@ -27,7 +27,7 @@ if [[ ${MLC_DATASET_CALIBRATION} == "no" ]]; then eval $cmd test $? -eq 0 || exit $? else - cmd="./download-coco-2014-calibration.sh -d ${INSTALL_DIR}" + cmd="./download-coco-2014-calibration.sh -d ${INSTALL_DIR} -n ${MLC_DATASET_COCO2014_NUM_WORKERS}" echo $cmd eval $cmd test $? -eq 0 || exit $? From 854c511ae8965879db1ab9b18cca656e4c7ca1b7 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 4 Mar 2025 14:15:04 +0530 Subject: [PATCH 04/27] Update script to detect Podman in the system (#287) * Add support for podman * [Automated Commit] Format Codebase [skip ci] * Container tool name made dynamic --- script/get-docker/customize.py | 33 ++++++++++++++++++++++++++------- script/get-docker/run.sh | 2 +- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/script/get-docker/customize.py b/script/get-docker/customize.py index c8aaf7376..3097e343f 100644 --- a/script/get-docker/customize.py +++ b/script/get-docker/customize.py @@ -12,11 +12,15 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - file_name = 'docker.exe' if os_info['platform'] == 'windows' else 'docker' - env['FILE_NAME'] = file_name + file_name_docker = 'docker.exe' if os_info['platform'] == 'windows' else 'docker' + file_name_podman = 'podman.exe' if os_info['platform'] == 'windows' else 'podman' if 'MLC_DOCKER_BIN_WITH_PATH' not in env: - r = i['automation'].find_artifact({'file_name': file_name, + # check for docker + # if docker is not found, podman is checked + env['FILE_NAME'] = file_name_docker + env['CONTAINER_TOOL_NAME'] = "docker" + r = i['automation'].find_artifact({'file_name': file_name_docker, 'env': env, 'os_info': os_info, 'default_path_env_key': 'PATH', @@ -26,11 +30,26 @@ def preprocess(i): 'recursion_spaces': recursion_spaces}) if r['return'] > 0: if r['return'] == 16: - run_file_name = "install" - r = automation.run_native_script( - {'run_script_input': i['run_script_input'], 'env': env, 'script_name': run_file_name}) + # check for podman + # if podman is also absent, the script will try to + # automatically install docker in the system + env['FILE_NAME'] = file_name_podman + env['CONTAINER_TOOL_NAME'] = "podman" + r = i['automation'].find_artifact({'file_name': file_name_podman, + 'env': env, + 'os_info': os_info, + 'default_path_env_key': 'PATH', + 'detect_version': True, + 'env_path_key': 'MLC_DOCKER_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) if r['return'] > 0: - return r + if r['return'] == 16: + run_file_name = "install" + r = automation.run_native_script( + {'run_script_input': i['run_script_input'], 'env': env, 'script_name': run_file_name}) + if r['return'] > 0: + return r else: return r diff --git a/script/get-docker/run.sh b/script/get-docker/run.sh index f7f946a7f..f78518097 100644 --- a/script/get-docker/run.sh +++ b/script/get-docker/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker --version > tmp-ver.out +${CONTAINER_TOOL_NAME} --version > tmp-ver.out test $? -eq 0 || exit 1 From 1fb2579c86d2d212f346766e3d4cc74ca4713ec5 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Wed, 5 Mar 2025 12:10:27 +0530 Subject: [PATCH 05/27] Added Resnet50 closed division github action Fix for #247 --- ...erf-inference-resnet50-closed-division.yml | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/test-mlperf-inference-resnet50-closed-division.yml diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml new file mode 100644 index 000000000..0822b7f37 --- /dev/null +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -0,0 +1,133 @@ +name: MLPerf inference ResNet50 Closed Division for testing Compliance + +on: + pull_request_target: + branches: [ "main", "dev" ] + paths: + - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' + - '**' + - '!**.md' +jobs: + build: + name: MLPerf inference MLCommons ResNet50 Closed Division + runs-on: ${{ matrix.os }} + env: + MLC_INDEX: "on" + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.12" ] + backend: [ "onnxruntime", "tf" ] + implementation: [ "python", "cpp" ] + exclude: + - backend: tf + implementation: cpp + - os: macos-latest + backend: tf + - os: windows-latest + implementation: cpp + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check if the submission is for closed division + run: | + if git diff --name-only origin/${{ github.base_ref }}..HEAD | xargs grep -l "division=closed"; then + echo "DIVISION_CLOSED=true" >> $GITHUB_ENV + else + echo "DIVISION_CLOSED=false" >> $GITHUB_ENV + fi + + - name: Check value for the divison + run: echo "DIVISION_CLOSED=${{ env.DIVISION_CLOSED }}" + + - name: Skip workflow if its an open division submission as no compliance test is required + if: ${{ env.DIVISION_CLOSED == 'false' }} + run: | + echo "Skipping workflow because division=closed is not found in the changes." + exit 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Configure git longpaths (Windows) + if: matrix.os == 'windows-latest' + run: | + git config --system core.longpaths true + + - name: Install mlcflow + run: | + pip install mlcflow + pip install tabulate + + - name: Pull MLOps repo + run: | + mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + - name: Test MLPerf Inference ResNet50 (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet + + - name: Compliance Test MLPerf Inference ResNet50 (Windows) + if: matrix.os == 'windows-latest' + run: mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + + - name: Test MLPerf Inference ResNet50 (Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + + - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) + run: | + mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes + + + # Step for Linux/MacOS + - name: Randomly Execute Step (Linux/MacOS) + if: runner.os != 'Windows' + run: | + RANDOM_NUMBER=$((RANDOM % 10)) + echo "Random number is $RANDOM_NUMBER" + if [ "$RANDOM_NUMBER" -eq 0 ]; then + echo "run_step=true" >> $GITHUB_ENV + else + echo "run_step=false" >> $GITHUB_ENV + fi + + # Step for Windows + - name: Randomly Execute Step (Windows) + if: runner.os == 'Windows' + run: | + $RANDOM_NUMBER = Get-Random -Maximum 10 + Write-Host "Random number is $RANDOM_NUMBER" + if ($RANDOM_NUMBER -eq 0) { + Write-Host "run_step=true" | Out-File -FilePath $Env:GITHUB_ENV -Append + } else { + Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append + } + + - name: Retrieve secrets from Keeper + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + id: ksecrets + uses: Keeper-Security/ksm-action@master + with: + keeper-secret-config: ${{ secrets.KSM_CONFIG }} + secrets: |- + ubwkjh-Ii8UJDpG2EoU6GQ/field/Access Token > env:PAT + - name: Push Results + env: + GITHUB_TOKEN: ${{ env.PAT }} + if: github.repository_owner == 'mlcommons' && env.run_step == 'true' + run: | + git config --global user.name "mlcommons-bot" + git config --global user.email "mlcommons-bot@users.noreply.github.com" + git config --global credential.https://github.com.helper "" + git config --global credential.https://github.com.helper "!gh auth git-credential" + git config --global credential.https://gist.github.com.helper "" + git config --global credential.https://gist.github.com.helper "!gh auth git-credential" + mlcr push,github,mlperf,inference,submission --repo_url=https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0 --repo_branch=auto-update --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet + From 234aad7aa480cbd63d06c0234ef2dc0a353078d8 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Wed, 5 Mar 2025 12:22:36 +0530 Subject: [PATCH 06/27] Scheduling the workflow once a week --- .../test-mlperf-inference-resnet50-closed-division.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 0822b7f37..9c98f773c 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -1,6 +1,8 @@ name: MLPerf inference ResNet50 Closed Division for testing Compliance on: + schedule: + - cron: '0 0 * * 0' # Runs once a week pull_request_target: branches: [ "main", "dev" ] paths: From a5b059617f3905cd8626dab9fde53ca3136034c6 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 5 Mar 2025 16:17:33 +0530 Subject: [PATCH 07/27] Fix aocc detect --- script/get-aocc/customize.py | 20 +++++--------------- script/get-aocc/meta.yaml | 1 + 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 383cac39f..106c4674c 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -20,23 +20,13 @@ def preprocess(i): exe_c = 'clang.exe' if os_info['platform'] == 'windows' else 'clang' - if env.get('MLC_AOCC_DIR_PATH', '') != '' and env.get( - 'MLC_AOCC_BIN_WITH_PATH', '') == '': - for f in os.listdir(env['MLC_AOCC_DIR_PATH']): - if os.path.exists(os.path.join( - env['MLC_AOCC_DIR_PATH'], f, "bin", exe_c)): - env['MLC_AOCC_BIN_WITH_PATH'] = os.path.join( - env['MLC_AOCC_DIR_PATH'], f, "bin", exe_c) - - if env.get('MLC_HOST_OS_FLAVOR', '') == 'rhel': - if "12" in env.get('MLC_VERSION', '') or "12" in env.get( - 'MLC_VERSION_MIN', ''): - if env.get('MLC_TMP_PATH', '') == '': - env['MLC_TMP_PATH'] = '' - env['MLC_TMP_PATH'] += "/opt/rh/aocc/root/usr/bin" - env['MLC_TMP_PATH_IGNORE_NON_EXISTANT'] = 'yes' if 'MLC_AOCC_BIN_WITH_PATH' not in env: + if env.get('MLC_AOCC_DIR_PATH', '') != '': + aocc_path = env['MLC_AOCC_DIR_PATH'] + if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')): + env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') + r = i['automation'].find_artifact({'file_name': exe_c, 'env': env, 'os_info': os_info, diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index 80a21d311..429a9ba95 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -20,6 +20,7 @@ deps: input_mapping: tar_file_path: MLC_AOCC_TAR_FILE_PATH + aocc_dir: MLC_AOCC_DIR_PATH name: Detect or install AOCC compiler new_env_keys: From 6ae37749bbde7fa902f7f645895434f8fde5b368 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 5 Mar 2025 10:49:30 +0000 Subject: [PATCH 08/27] [Automated Commit] Format Codebase [skip ci] --- script/get-aocc/customize.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index 106c4674c..ba03e389f 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -20,13 +20,12 @@ def preprocess(i): exe_c = 'clang.exe' if os_info['platform'] == 'windows' else 'clang' - if 'MLC_AOCC_BIN_WITH_PATH' not in env: if env.get('MLC_AOCC_DIR_PATH', '') != '': aocc_path = env['MLC_AOCC_DIR_PATH'] if os.path.exists(os.path.join(aocc_path, 'bin', 'clang')): env['MLC_TMP_PATH'] = os.path.join(aocc_path, 'bin') - + r = i['automation'].find_artifact({'file_name': exe_c, 'env': env, 'os_info': os_info, From beda189cf1f1058454fe3a1c2a0242478ddb0386 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Thu, 6 Mar 2025 07:12:21 +0530 Subject: [PATCH 09/27] Changed the dataset parameter to short --- .../test-mlperf-inference-resnet50-closed-division.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 9c98f773c..1798ac9e8 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -9,6 +9,7 @@ on: - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - '**' - '!**.md' + workflow_dispatch: # Allows manual triggering of the workflow jobs: build: name: MLPerf inference MLCommons ResNet50 Closed Division @@ -72,20 +73,20 @@ jobs: - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet - name: Compliance Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' - run: mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + run: mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes - name: Test MLPerf Inference ResNet50 (Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) run: | - mlcr run-mlperf,inference,_submission,_full, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes + mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes # Step for Linux/MacOS From 0c166558412da632b63b6e349935aa5138770ac6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Mar 2025 23:20:06 +0000 Subject: [PATCH 10/27] Merge from GO (#292) * Invalidate invalid config results * Fix v25.02 for armnn * Fix git push command if working directory is clean * Added _path.# variations for gcc,aocc --- script/generate-mlperf-inference-user-conf/customize.py | 1 + script/get-aocc/meta.yaml | 4 ++++ script/get-gcc/customize.py | 6 ++++++ script/get-gcc/meta.yaml | 4 ++++ script/get-lib-armnn/meta.yaml | 4 ++++ script/push-mlperf-inference-results-to-github/run.sh | 9 +++++++-- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-user-conf/customize.py b/script/generate-mlperf-inference-user-conf/customize.py index a9f8e3eaa..70d920ed4 100644 --- a/script/generate-mlperf-inference-user-conf/customize.py +++ b/script/generate-mlperf-inference-user-conf/customize.py @@ -480,6 +480,7 @@ def run_files_exist(mode, OUTPUT_DIR, run_files, env): if ( "result_validity" not in mlperf_log.get_keys() or mlperf_log["result_validity"] != "VALID" + or "error_invalid_config" in mlperf_log.get_keys() ): return False diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index 429a9ba95..c620a5d03 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -43,3 +43,7 @@ tags: - get - aocc uid: 1ceb0656e99a44ec +variations: + _path.#: + env: + MLC_AOCC_DIR_PATH: # diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index a8c8b3099..01749aba5 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -11,6 +11,12 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] file_name_c = 'gcc.exe' if os_info['platform'] == 'windows' else 'gcc' + if 'MLC_GCC_BIN_WITH_PATH' not in env: + if env.get('MLC_GCC_DIR_PATH', '') != '': + gcc_path = env['MLC_GCC_DIR_PATH'] + if os.path.exists(os.path.join(gcc_path, 'bin', 'gcc')): + env['MLC_TMP_PATH'] = os.path.join(gcc_path, 'bin') + if env.get('MLC_HOST_OS_FLAVOR', '') == 'rhel': if "12" in env.get('MLC_VERSION', '') or "12" in env.get( 'MLC_VERSION_MIN', ''): diff --git a/script/get-gcc/meta.yaml b/script/get-gcc/meta.yaml index 27a3b6feb..4de1812f5 100644 --- a/script/get-gcc/meta.yaml +++ b/script/get-gcc/meta.yaml @@ -30,3 +30,7 @@ tags: - cpp-compiler - get-gcc uid: dbf4ab5cbed74372 +variations: + _path.#: + env: + MLC_GCC_DIR_PATH: # diff --git a/script/get-lib-armnn/meta.yaml b/script/get-lib-armnn/meta.yaml index ff71bf95a..53b5e210d 100644 --- a/script/get-lib-armnn/meta.yaml +++ b/script/get-lib-armnn/meta.yaml @@ -37,3 +37,7 @@ versions: env: MLC_LIB_ARMNN_VERSION: v23.11 MLC_TMP_GIT_BRANCH_NAME: branches/armnn_23_11 + '25.02': + env: + MLC_LIB_ARMNN_VERSION: v23.11 + MLC_TMP_GIT_BRANCH_NAME: branches/armnn_23_11 diff --git a/script/push-mlperf-inference-results-to-github/run.sh b/script/push-mlperf-inference-results-to-github/run.sh index ffac61801..5994b1068 100644 --- a/script/push-mlperf-inference-results-to-github/run.sh +++ b/script/push-mlperf-inference-results-to-github/run.sh @@ -16,8 +16,13 @@ if [[ -n ${MLC_MLPERF_INFERENCE_SUBMISSION_DIR} ]]; then fi test $? -eq 0 || exit $? -git commit -a -m "${MLC_MLPERF_RESULTS_REPO_COMMIT_MESSAGE}" -test $? -eq 0 || exit $? +if ! git diff-index --quiet HEAD --; then + git commit -a -m "${MLC_MLPERF_RESULTS_REPO_COMMIT_MESSAGE}" + test $? -eq 0 || exit $? +else + echo "No changes to commit." +fi + echo ${MLC_GIT_PUSH_CMD} ${MLC_GIT_PUSH_CMD} From ea48b11b8819352d36e97252d56d4f068128fa74 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Fri, 7 Mar 2025 10:03:01 +0530 Subject: [PATCH 11/27] Fixes for test-mlpref-inference-resnet50-closed-division.yml --- ...erf-inference-resnet50-closed-division.yml | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 1798ac9e8..574d0ae19 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -1,6 +1,9 @@ name: MLPerf inference ResNet50 Closed Division for testing Compliance on: + # push: + # branches: + # - test schedule: - cron: '0 0 * * 0' # Runs once a week pull_request_target: @@ -9,7 +12,7 @@ on: - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - '**' - '!**.md' - workflow_dispatch: # Allows manual triggering of the workflow + # workflow_dispatch: # Allows manual triggering of the workflow jobs: build: name: MLPerf inference MLCommons ResNet50 Closed Division @@ -35,23 +38,6 @@ jobs: with: fetch-depth: 0 - - name: Check if the submission is for closed division - run: | - if git diff --name-only origin/${{ github.base_ref }}..HEAD | xargs grep -l "division=closed"; then - echo "DIVISION_CLOSED=true" >> $GITHUB_ENV - else - echo "DIVISION_CLOSED=false" >> $GITHUB_ENV - fi - - - name: Check value for the divison - run: echo "DIVISION_CLOSED=${{ env.DIVISION_CLOSED }}" - - - name: Skip workflow if its an open division submission as no compliance test is required - if: ${{ env.DIVISION_CLOSED == 'false' }} - run: | - echo "Skipping workflow because division=closed is not found in the changes." - exit 0 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: @@ -69,25 +55,46 @@ jobs: - name: Pull MLOps repo run: | mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} + + + - name: Test MLPerf Inference ResNet50 Offline (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - name: Test MLPerf Inference ResNet50 (Windows) + - name: Test MLPerf Inference ResNet50 Singlestream (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet - + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --target_latency=10 + + - name: Test MLPerf Inference ResNet50 Multistream (Windows) + if: matrix.os == 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream , --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Compliance Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' - run: mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --quiet --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + run: mlcr run-mlperf, inference, _submission, _short,_all-scenarios, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes + + - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - name: Test MLPerf Inference ResNet50 (Linux/macOS) + - name: Test MLPerf Inference ResNet50 SingleStream(Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet + mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Test MLPerf Inference ResNet50 MultiStream(Linux/macOS) + if: matrix.os != 'windows-latest' + run: | + mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 + - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) + if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf,inference,_submission,_short,_all-scenarios, division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet --compliance=yes - + mlcr run-mlperf, inference, _submission, _short, _all-scenarios --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 -v --quiet --execution_mode=valid --compliance=yes # Step for Linux/MacOS - name: Randomly Execute Step (Linux/MacOS) From adec3319475ad62a83b87aa2d20397584313f708 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 8 Mar 2025 01:42:31 +0530 Subject: [PATCH 12/27] Add get-oneapi compiler(WIP) (#294) * initial commit for oneapi * fix group number * [Automated Commit] Format Codebase [skip ci] * code fixes * [Automated Commit] Format Codebase [skip ci] --- script/get-one-api/COPYRIGHT.md | 9 +++ script/get-one-api/customize.py | 99 +++++++++++++++++++++++++++++++++ script/get-one-api/meta.yaml | 27 +++++++++ script/get-one-api/run.bat | 3 + script/get-one-api/run.sh | 8 +++ 5 files changed, 146 insertions(+) create mode 100644 script/get-one-api/COPYRIGHT.md create mode 100644 script/get-one-api/customize.py create mode 100644 script/get-one-api/meta.yaml create mode 100644 script/get-one-api/run.bat create mode 100644 script/get-one-api/run.sh diff --git a/script/get-one-api/COPYRIGHT.md b/script/get-one-api/COPYRIGHT.md new file mode 100644 index 000000000..d2ceead84 --- /dev/null +++ b/script/get-one-api/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +© 2025-2026 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-one-api/customize.py b/script/get-one-api/customize.py new file mode 100644 index 000000000..1dade8a51 --- /dev/null +++ b/script/get-one-api/customize.py @@ -0,0 +1,99 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + recursion_spaces = i['recursion_spaces'] + file_name_c = 'oneapi.exe' if os_info['platform'] == 'windows' else 'oneapi-cli' + + if 'MLC_ONEAPI_BIN_WITH_PATH' not in env: + if env.get('MLC_ONEAPI_DIR_PATH', '') != '': + oneapi_path = env['MLC_ONEAPI_DIR_PATH'] + if os.path.exists(os.path.join(oneapi_path, 'bin', 'oneapi-cli')): + env['MLC_TMP_PATH'] = os.path.join(oneapi_path, 'bin') + + if 'MLC_ONEAPI_BIN_WITH_PATH' not in env: + r = i['automation'].find_artifact({'file_name': file_name_c, + 'env': env, + 'os_info': os_info, + 'default_path_env_key': 'PATH', + 'detect_version': True, + 'env_path_key': 'MLC_ONEAPI_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) + if r['return'] > 0: + # Uncomment when MLC script for installing oneapi compiler is integrated + # Initial finding suggests that oneapi could be installed without + # hastle in linux using apt, but is tricky in windows + + # if r['return'] == 16: + # if env.get('MLC_TMP_FAIL_IF_NOT_FOUND','').lower() == 'yes': + # return r + # + # print (recursion_spaces+' # {}'.format(r['error'])) + # + # # Attempt to run installer + # r = {'return':0, 'skip':True, 'script':{'tags':'install,gcc,src'}} + + return r + + return {'return': 0} + + +def detect_version(i): + r = i['automation'].parse_version({'match_text': r'^.*$', + 'group_number': 0, + 'env_key': 'MLC_ONEAPI_VERSION', + 'which_env': i['env']}) + if r['return'] > 0: + return r + + version = r['version'] + + print(i['recursion_spaces'] + ' Detected version: {}'.format(version)) + + return {'return': 0, 'version': version} + + +def postprocess(i): + + env = i['env'] + r = detect_version(i) + if r['return'] > 0: + return r + + env['MLC_COMPILER_FAMILY'] = 'ONEAPI' + version = r['version'] + env['MLC_COMPILER_VERSION'] = env['MLC_ONEAPI_VERSION'] + env['MLC_GCC_CACHE_TAGS'] = 'version-' + version + env['MLC_COMPILER_CACHE_TAGS'] = 'version-' + version + ',family-oneapi' + + found_file_path = env['MLC_ONEAPI_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + + env['MLC_ONEAPI_INSTALLED_PATH'] = found_path + + file_name_c = os.path.basename(found_file_path) + + env['MLC_ONEAPI_BIN'] = file_name_c + + # General compiler for general program compilation + env['MLC_ONEAPI_COMPILER_BIN'] = file_name_c + env['MLC_ONEAPI_COMPILER_FLAG_OUTPUT'] = '' + env['MLC_ONEAPI_COMPILER_WITH_PATH'] = found_file_path + env['MLC_ONEAPI_COMPILER_FLAG_VERSION'] = 'version' + + # env['MLC_COMPILER_FLAGS_FAST'] = "-O3" + # env['MLC_LINKER_FLAGS_FAST'] = "-O3" + # env['MLC_COMPILER_FLAGS_DEBUG'] = "-O0" + # env['MLC_LINKER_FLAGS_DEBUG'] = "-O0" + # env['MLC_COMPILER_FLAGS_DEFAULT'] = "-O2" + # env['MLC_LINKER_FLAGS_DEFAULT'] = "-O2" + + return {'return': 0, 'version': version} diff --git a/script/get-one-api/meta.yaml b/script/get-one-api/meta.yaml new file mode 100644 index 000000000..edbc993b4 --- /dev/null +++ b/script/get-one-api/meta.yaml @@ -0,0 +1,27 @@ +alias: get-one-api +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: Compiler automation +clean_files: [] +deps: +- tags: detect,os +name: Detect or install OneAPI compiler +new_env_keys: +- MLC_ONEAPI_* +- MLC_ONEAPI_COMPILER_* +- MLC_COMPILER_* +- MLC_LINKER_* +post_deps: +- tags: get,compiler-flags +sort: 500 +tags: +- get +- oneapi +- compiler +- get-oneapi +uid: 1af872e81ef54742 +variations: + _path.#: + env: + MLC_ONEAPI_DIR_PATH: null diff --git a/script/get-one-api/run.bat b/script/get-one-api/run.bat new file mode 100644 index 000000000..5c02e5332 --- /dev/null +++ b/script/get-one-api/run.bat @@ -0,0 +1,3 @@ +%MLC_GCC_BIN_WITH_PATH% version > tmp-ver.out +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% + diff --git a/script/get-one-api/run.sh b/script/get-one-api/run.sh new file mode 100644 index 000000000..5cb2fa044 --- /dev/null +++ b/script/get-one-api/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +oneapi_bin=${MLC_ONEAPI_BIN_WITH_PATH} +echo "${oneapi_bin} version" + +${oneapi_bin} version > tmp-ver.out +test $? -eq 0 || exit 1 + +cat tmp-ver.out From aefdde0fbaaf4542a6c733bb56a120e70f48053c Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 8 Mar 2025 02:08:30 +0530 Subject: [PATCH 13/27] Fixes oneapi detect --- script/get-aocc/customize.py | 3 ++- script/get-aocc/meta.yaml | 2 +- script/get-one-api/customize.py | 19 ++++++++++--------- script/get-one-api/meta.yaml | 8 ++++++-- script/get-one-api/run.sh | 10 ++++------ 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index ba03e389f..354a34d80 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -72,7 +72,8 @@ def postprocess(i): found_path = os.path.dirname(found_file_path) - env['MLC_AOCC_INSTALLED_PATH'] = found_path + env['MLC_AOCC_BIN_PATH'] = found_path + env['MLC_AOCC_INSTALLED_PATH'] = os.path.dirname(found_path) file_name_c = os.path.basename(found_file_path) file_name_cpp = file_name_c.replace('clang', 'clang++') diff --git a/script/get-aocc/meta.yaml b/script/get-aocc/meta.yaml index c620a5d03..ff12c56ec 100644 --- a/script/get-aocc/meta.yaml +++ b/script/get-aocc/meta.yaml @@ -35,7 +35,7 @@ new_env_keys: - + LDFLAGS - +MLC_HOST_OS_DEFAULT_INCLUDE_PATH - +PATH -post_depsq: +post_deps_off: - tags: get,compiler-flags sort: 500 tags: diff --git a/script/get-one-api/customize.py b/script/get-one-api/customize.py index 1dade8a51..0f6e78b10 100644 --- a/script/get-one-api/customize.py +++ b/script/get-one-api/customize.py @@ -1,5 +1,6 @@ from mlc import utils import os +from utils import * def preprocess(i): @@ -9,12 +10,12 @@ def preprocess(i): env = i['env'] recursion_spaces = i['recursion_spaces'] - file_name_c = 'oneapi.exe' if os_info['platform'] == 'windows' else 'oneapi-cli' + file_name_c = 'icx.exe' if os_info['platform'] == 'windows' else 'icx' - if 'MLC_ONEAPI_BIN_WITH_PATH' not in env: + if 'MLC_ICX_BIN_WITH_PATH' not in env: if env.get('MLC_ONEAPI_DIR_PATH', '') != '': oneapi_path = env['MLC_ONEAPI_DIR_PATH'] - if os.path.exists(os.path.join(oneapi_path, 'bin', 'oneapi-cli')): + if os.path.exists(os.path.join(oneapi_path, 'bin', 'icx')): env['MLC_TMP_PATH'] = os.path.join(oneapi_path, 'bin') if 'MLC_ONEAPI_BIN_WITH_PATH' not in env: @@ -23,7 +24,7 @@ def preprocess(i): 'os_info': os_info, 'default_path_env_key': 'PATH', 'detect_version': True, - 'env_path_key': 'MLC_ONEAPI_BIN_WITH_PATH', + 'env_path_key': 'MLC_ICX_BIN_WITH_PATH', 'run_script_input': i['run_script_input'], 'recursion_spaces': recursion_spaces}) if r['return'] > 0: @@ -46,8 +47,8 @@ def preprocess(i): def detect_version(i): - r = i['automation'].parse_version({'match_text': r'^.*$', - 'group_number': 0, + r = i['automation'].parse_version({'match_text': r'oneAPI\s+.*\(([\d.]+)\)', + 'group_number': 1, 'env_key': 'MLC_ONEAPI_VERSION', 'which_env': i['env']}) if r['return'] > 0: @@ -70,14 +71,14 @@ def postprocess(i): env['MLC_COMPILER_FAMILY'] = 'ONEAPI' version = r['version'] env['MLC_COMPILER_VERSION'] = env['MLC_ONEAPI_VERSION'] - env['MLC_GCC_CACHE_TAGS'] = 'version-' + version + env['MLC_ONEAPI_CACHE_TAGS'] = 'version-' + version env['MLC_COMPILER_CACHE_TAGS'] = 'version-' + version + ',family-oneapi' - found_file_path = env['MLC_ONEAPI_BIN_WITH_PATH'] + found_file_path = env['MLC_ICX_BIN_WITH_PATH'] found_path = os.path.dirname(found_file_path) - env['MLC_ONEAPI_INSTALLED_PATH'] = found_path + env['MLC_ONEAPI_INSTALLED_PATH'] = os.path.dirname(found_path) file_name_c = os.path.basename(found_file_path) diff --git a/script/get-one-api/meta.yaml b/script/get-one-api/meta.yaml index edbc993b4..1033eb772 100644 --- a/script/get-one-api/meta.yaml +++ b/script/get-one-api/meta.yaml @@ -12,7 +12,11 @@ new_env_keys: - MLC_ONEAPI_COMPILER_* - MLC_COMPILER_* - MLC_LINKER_* -post_deps: + +input_mapping: + oneapi_dir: MLC_ONEAPI_DIR_PATH + +post_deps_off: - tags: get,compiler-flags sort: 500 tags: @@ -24,4 +28,4 @@ uid: 1af872e81ef54742 variations: _path.#: env: - MLC_ONEAPI_DIR_PATH: null + MLC_ONEAPI_DIR_PATH: "#" diff --git a/script/get-one-api/run.sh b/script/get-one-api/run.sh index 5cb2fa044..756cc54e1 100644 --- a/script/get-one-api/run.sh +++ b/script/get-one-api/run.sh @@ -1,8 +1,6 @@ #!/bin/bash -oneapi_bin=${MLC_ONEAPI_BIN_WITH_PATH} -echo "${oneapi_bin} version" +icx_bin=${MLC_ICX_BIN_WITH_PATH} +echo "${icx_bin} --version" -${oneapi_bin} version > tmp-ver.out -test $? -eq 0 || exit 1 - -cat tmp-ver.out +${icx_bin} --version > tmp-ver.out +test $? -eq 0 || exit $? From b073ced646f7d2b2fc70aa3da2114a42bed69dca Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 8 Mar 2025 18:13:35 +0000 Subject: [PATCH 14/27] Support OS_FLAVOR_LIKE and OS_TYPE in run scripts (#297) * Support OS_FLAVOR_LIKE and OS_TYPE in run script selection * [Automated Commit] Format Codebase [skip ci] * Added get-wkhtmltopdf script * Added run.bat for get-wkhtmltopdf --- automation/script/module.py | 69 +++++++++++++++++++--------- script/get-wkhtmltopdf/meta.yaml | 17 +++++++ script/get-wkhtmltopdf/run-fedora.sh | 16 +++++++ script/get-wkhtmltopdf/run-ubuntu.sh | 5 ++ script/get-wkhtmltopdf/run.bat | 30 ++++++++++++ 5 files changed, 115 insertions(+), 22 deletions(-) create mode 100755 script/get-wkhtmltopdf/meta.yaml create mode 100644 script/get-wkhtmltopdf/run-fedora.sh create mode 100644 script/get-wkhtmltopdf/run-ubuntu.sh create mode 100644 script/get-wkhtmltopdf/run.bat diff --git a/automation/script/module.py b/automation/script/module.py index acdacff8e..731d3b075 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -5565,29 +5565,54 @@ def run_postprocess(customize_code, customize_common_input, recursion_spaces, def get_script_name(env, path, script_name='run'): - """ - Internal: find the most appropriate run script name for the detected OS - """ + # Extract environment variables safely, defaulting to empty strings if + # missing + os_flavor = env.get('MLC_HOST_OS_FLAVOR', '') + os_flavor_like = env.get('MLC_HOST_OS_FLAVOR_LIKE', '') + os_type = env.get('MLC_HOST_OS_TYPE', '') + # Only use version if flavor exists + os_version = env.get('MLC_HOST_OS_VERSION', '') if os_flavor else '' + platform_flavor = env.get('MLC_HOST_PLATFORM_FLAVOR', '') + + # Get a list of all files in the directory + try: + available_files = set(os.listdir(path)) + except FileNotFoundError: + # Default if directory doesn't exist + return os.path.join(path, f"{script_name}.sh") + + # Check if any script with a "script_name-" prefix exists + has_prefixed_scripts = any(f.startswith( + f"{script_name}-") for f in available_files) + + # Helper function to construct script filenames dynamically + def script_filename(*parts): + # Remove empty values to avoid extra '-' + suffix = "-".join(filter(None, parts)) + return f"{script_name}-{suffix}.sh" if suffix else f"{script_name}.sh" + + # Define file search order based on priority + candidates = [ + script_filename(os_flavor, os_version, platform_flavor), + script_filename(os_flavor, os_version), + script_filename(os_flavor, platform_flavor), + script_filename(os_flavor), + script_filename(os_flavor_like, platform_flavor), + script_filename(os_flavor_like), + script_filename(os_type, platform_flavor), + script_filename(os_type), + script_filename(platform_flavor), + ] + + # If prefixed scripts exist, check for the first matching candidate + if has_prefixed_scripts: + for candidate in candidates: + if candidate in available_files: + return os.path.join(path, candidate) + + # Fallback to the default script + return os.path.join(path, f"{script_name}.sh") - from os.path import exists - - tmp_suff1 = env.get('MLC_HOST_OS_FLAVOR', '') - tmp_suff2 = env.get('MLC_HOST_OS_VERSION', '') - tmp_suff3 = env.get('MLC_HOST_PLATFORM_FLAVOR', '') - - if exists(os.path.join(path, script_name + '-' + tmp_suff1 + - '-' + tmp_suff2 + '-' + tmp_suff3 + '.sh')): - return script_name + '-' + tmp_suff1 + '-' + tmp_suff2 + '-' + tmp_suff3 + '.sh' - elif exists(os.path.join(path, script_name + '-' + tmp_suff1 + '-' + tmp_suff3 + '.sh')): - return script_name + '-' + tmp_suff1 + '-' + tmp_suff3 + '.sh' - elif exists(os.path.join(path, script_name + '-' + tmp_suff1 + '-' + tmp_suff2 + '.sh')): - return script_name + '-' + tmp_suff1 + '-' + tmp_suff2 + '.sh' - elif exists(os.path.join(path, script_name + '-' + tmp_suff1 + '.sh')): - return script_name + '-' + tmp_suff1 + '.sh' - elif exists(os.path.join(path, script_name + '-' + tmp_suff3 + '.sh')): - return script_name + '-' + tmp_suff3 + '.sh' - else: - return script_name + '.sh' ############################################################################## diff --git a/script/get-wkhtmltopdf/meta.yaml b/script/get-wkhtmltopdf/meta.yaml new file mode 100755 index 000000000..eca50da63 --- /dev/null +++ b/script/get-wkhtmltopdf/meta.yaml @@ -0,0 +1,17 @@ +alias: get-wkhtmltopdf +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +tags: +- get +- wkhtmltopdf +deps: + - tags: detect,os + - tags: detect,sudo +uid: 67ec874a3dfe4b87 +variations: + with-qt: + group: qt + default: true + + diff --git a/script/get-wkhtmltopdf/run-fedora.sh b/script/get-wkhtmltopdf/run-fedora.sh new file mode 100644 index 000000000..cd3c713a6 --- /dev/null +++ b/script/get-wkhtmltopdf/run-fedora.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# This script installs wkhtmltopdf on Amazon Linux + +# Download the wkhtmltopdf package +wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.centos6.x86_64.rpm + +# Install the package +sudo yum localinstall -y wkhtmltox-0.12.6-1.centos6.x86_64.rpm + +# Install dependencies +sudo yum install -y xorg-x11-fonts-75dpi + +# Verify the installation +wkhtmltopdf --version + + diff --git a/script/get-wkhtmltopdf/run-ubuntu.sh b/script/get-wkhtmltopdf/run-ubuntu.sh new file mode 100644 index 000000000..13a1c642a --- /dev/null +++ b/script/get-wkhtmltopdf/run-ubuntu.sh @@ -0,0 +1,5 @@ +#!/bin/bash +wget -nc https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb +test $? -eq 0 || exit $? +${MLC_SUDO} dpkg -i wkhtmltox_0.12.6.1-2.jammy_amd64.deb +test $? -eq 0 || exit $? diff --git a/script/get-wkhtmltopdf/run.bat b/script/get-wkhtmltopdf/run.bat new file mode 100644 index 000000000..a172d0b66 --- /dev/null +++ b/script/get-wkhtmltopdf/run.bat @@ -0,0 +1,30 @@ +@echo off +setlocal + +:: Define download URL and filename +set "URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox-0.12.6-1.msvc2015-win64.exe" +set "FILE=wkhtmltox-0.12.6-1.msvc2015-win64.exe" + +:: Download the installer if it doesn't already exist +if not exist "%FILE%" ( + echo Downloading %FILE%... + powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%URL%', '%CD%\%FILE%')" + if %ERRORLEVEL% neq 0 ( + echo Download failed! + exit /b %ERRORLEVEL% + ) +) + +:: Install the software (silent mode) +echo Installing wkhtmltopdf... +start /wait %FILE% /S + +if %ERRORLEVEL% neq 0 ( + echo Installation failed! + exit /b %ERRORLEVEL% +) + +echo Installation successful! + +endlocal +exit /b 0 From 30cfdbdbf641cd7be27ce42845be59c5ab27312f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 8 Mar 2025 21:54:49 +0000 Subject: [PATCH 15/27] Fixes for wkhtmltopdf (#299) --- script/get-generic-sys-util/meta.yaml | 11 +++++++++++ script/get-wkhtmltopdf/meta.yaml | 6 ++++++ script/get-wkhtmltopdf/run-fedora.sh | 9 ++------- script/get-wkhtmltopdf/run-ubuntu.sh | 2 ++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index 4516a506a..a3faa4254 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -743,6 +743,17 @@ variations: wkhtmltopdf: apt: wkhtmltopdf brew: wkhtmltopdf + xfonts-base: + env: + MLC_SYS_UTIL_NAME: xfonts_base + new_env_keys: + - MLC_XFONTS_BASE_VERSION + state: + xfonts_base: + apt: xfonts-base + dnf: xorg-x11-fonts-misc + yum: xorg-x11-fonts-misc + zypper: xorg-x11-fonts xz: env: MLC_SYS_UTIL_NAME: xz diff --git a/script/get-wkhtmltopdf/meta.yaml b/script/get-wkhtmltopdf/meta.yaml index eca50da63..3b3b13677 100755 --- a/script/get-wkhtmltopdf/meta.yaml +++ b/script/get-wkhtmltopdf/meta.yaml @@ -8,7 +8,13 @@ tags: deps: - tags: detect,os - tags: detect,sudo + skip_if_env: + MLC_HOST_OS_TYPE: + - windows uid: 67ec874a3dfe4b87 +docker: + pre_run_cmds: + - mlc pull repo variations: with-qt: group: qt diff --git a/script/get-wkhtmltopdf/run-fedora.sh b/script/get-wkhtmltopdf/run-fedora.sh index cd3c713a6..d91b05b34 100644 --- a/script/get-wkhtmltopdf/run-fedora.sh +++ b/script/get-wkhtmltopdf/run-fedora.sh @@ -5,12 +5,7 @@ wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.centos6.x86_64.rpm # Install the package -sudo yum localinstall -y wkhtmltox-0.12.6-1.centos6.x86_64.rpm +${MLC_SUDO} yum localinstall -y wkhtmltox-0.12.6-1.centos6.x86_64.rpm --nobest --skip-broken -# Install dependencies -sudo yum install -y xorg-x11-fonts-75dpi - -# Verify the installation wkhtmltopdf --version - - +test $? -eq 0 || exit $? diff --git a/script/get-wkhtmltopdf/run-ubuntu.sh b/script/get-wkhtmltopdf/run-ubuntu.sh index 13a1c642a..ebafe84ef 100644 --- a/script/get-wkhtmltopdf/run-ubuntu.sh +++ b/script/get-wkhtmltopdf/run-ubuntu.sh @@ -2,4 +2,6 @@ wget -nc https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb test $? -eq 0 || exit $? ${MLC_SUDO} dpkg -i wkhtmltox_0.12.6.1-2.jammy_amd64.deb +${MLC_SUDO} DEBIAN_FRONTEND=noninteractive apt-get install -f -y test $? -eq 0 || exit $? + From c11befcbee2696ad8a4171beec0b6b863aa2201a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 8 Mar 2025 22:29:41 +0000 Subject: [PATCH 16/27] Fix wkhtmltopdf on macos (#300) --- script/get-wkhtmltopdf/meta.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/script/get-wkhtmltopdf/meta.yaml b/script/get-wkhtmltopdf/meta.yaml index 3b3b13677..02465f50f 100755 --- a/script/get-wkhtmltopdf/meta.yaml +++ b/script/get-wkhtmltopdf/meta.yaml @@ -11,6 +11,10 @@ deps: skip_if_env: MLC_HOST_OS_TYPE: - windows + - tags: get,generic-sys-util,_wkhtmltopdf + enable_if_env: + MLC_HOST_OS_TYPE: + - darwin uid: 67ec874a3dfe4b87 docker: pre_run_cmds: @@ -18,6 +22,4 @@ docker: variations: with-qt: group: qt - default: true - - + default: true \ No newline at end of file From 97c759ec171620db9413cd1480c4bd343aa2d798 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 8 Mar 2025 22:42:47 +0000 Subject: [PATCH 17/27] Added macos install wkhtmltopdf (#301) --- script/get-wkhtmltopdf/meta.yaml | 6 +++--- script/get-wkhtmltopdf/run-macos.sh | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 script/get-wkhtmltopdf/run-macos.sh diff --git a/script/get-wkhtmltopdf/meta.yaml b/script/get-wkhtmltopdf/meta.yaml index 02465f50f..3e0ff443a 100755 --- a/script/get-wkhtmltopdf/meta.yaml +++ b/script/get-wkhtmltopdf/meta.yaml @@ -13,8 +13,8 @@ deps: - windows - tags: get,generic-sys-util,_wkhtmltopdf enable_if_env: - MLC_HOST_OS_TYPE: - - darwin + MLC_INSTALL_FROM_PACKAGE_MANAGER: + - on uid: 67ec874a3dfe4b87 docker: pre_run_cmds: @@ -22,4 +22,4 @@ docker: variations: with-qt: group: qt - default: true \ No newline at end of file + default: true diff --git a/script/get-wkhtmltopdf/run-macos.sh b/script/get-wkhtmltopdf/run-macos.sh new file mode 100644 index 000000000..b04695459 --- /dev/null +++ b/script/get-wkhtmltopdf/run-macos.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# This script installs wkhtmltopdf on Amazon Linux +curl -L -o wkhtmltox-0.12.6-1.macos-cocoa.pkg https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.macos-cocoa.pkg +${MLC_SUDO} installer -pkg wkhtmltox-0.12.6-1.macos-cocoa.pkg -target / + +wkhtmltopdf --version +test $? -eq 0 || exit $? From 63f68f32e77f8335c33003c0e723c6361550bdc2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 8 Mar 2025 23:04:29 +0000 Subject: [PATCH 18/27] Update test-mlc-script-features.yml (#302) * Update test-mlc-script-features.yml * Fix windows install for wkhtmltopdf --- .github/workflows/test-mlc-script-features.yml | 4 ++++ script/get-wkhtmltopdf/run.bat | 9 +++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index 05d62e2ae..a61f6b3c7 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -43,6 +43,8 @@ jobs: mlcr get,dataset,preprocessed,imagenet,_NHWC --quiet mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NCHW mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NHWC + # Tests run-scripts on windows,ubuntu,macos + mlcr get,wkhtmltopdf - name: Test versions continue-on-error: true @@ -52,6 +54,8 @@ jobs: test $? -eq 0 || exit $? mlcr get,generic-python-lib,_package.scipy --version=1.9.2 --quiet test $? -eq 0 || exit $? + mlc find cache --tags=get,generic-python-lib,_package.scipy,version-1.9.3 + test $? -eq 0 || exit $? # Need to add find cache here # mlcr get,generic-python-lib,_package.scipy --version=1.9.3 --quiet --only_execute_from_cache=True # test $? -eq 0 || exit 0 diff --git a/script/get-wkhtmltopdf/run.bat b/script/get-wkhtmltopdf/run.bat index a172d0b66..6ccb6b19a 100644 --- a/script/get-wkhtmltopdf/run.bat +++ b/script/get-wkhtmltopdf/run.bat @@ -1,14 +1,11 @@ -@echo off -setlocal - :: Define download URL and filename -set "URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox-0.12.6-1.msvc2015-win64.exe" +set "URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.msvc2015-win64.exe" set "FILE=wkhtmltox-0.12.6-1.msvc2015-win64.exe" :: Download the installer if it doesn't already exist if not exist "%FILE%" ( echo Downloading %FILE%... - powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%URL%', '%CD%\%FILE%')" + powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%URL%', '%CD%\\%FILE%')" if %ERRORLEVEL% neq 0 ( echo Download failed! exit /b %ERRORLEVEL% @@ -17,7 +14,7 @@ if not exist "%FILE%" ( :: Install the software (silent mode) echo Installing wkhtmltopdf... -start /wait %FILE% /S +start /wait %FILE% /quiet if %ERRORLEVEL% neq 0 ( echo Installation failed! From 826ec33e28e6137e51b8543cfba7c14b9ebb855e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 11:27:02 +0000 Subject: [PATCH 19/27] Fix mlc test script (#303) --- .github/scripts/process_individual_tests.py | 2 + .github/workflows/check-broken-links.yml | 2 - .../workflows/test-mlc-script-features.yml | 5 +- automation/script/module.py | 285 ++---------------- script/get-generic-sys-util/meta.yaml | 4 +- script/install-tensorflow-from-src/meta.yaml | 2 +- .../meta.yaml | 8 +- script/template-script/COPYRIGHT.md | 9 - 8 files changed, 44 insertions(+), 273 deletions(-) delete mode 100644 script/template-script/COPYRIGHT.md diff --git a/.github/scripts/process_individual_tests.py b/.github/scripts/process_individual_tests.py index a8df24239..708a80963 100644 --- a/.github/scripts/process_individual_tests.py +++ b/.github/scripts/process_individual_tests.py @@ -35,3 +35,5 @@ ret = mlc.access(ii) if ret['return'] > 0: raise Exception(r['error']) + ii = {'action': 'rm', 'target': 'cache', 'f': True} + ret = mlc.access(ii) diff --git a/.github/workflows/check-broken-links.yml b/.github/workflows/check-broken-links.yml index 06034c32a..0a07b08ec 100644 --- a/.github/workflows/check-broken-links.yml +++ b/.github/workflows/check-broken-links.yml @@ -1,5 +1,4 @@ name: "Check .md README files for broken links" - on: pull_request: branches: @@ -13,7 +12,6 @@ jobs: steps: - uses: actions/checkout@v4 - # Checks the status of hyperlinks in .md files in verbose mode - name: Check links uses: gaurav-nelson/github-action-markdown-link-check@v1 diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index a61f6b3c7..88708f577 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -44,7 +44,10 @@ jobs: mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NCHW mlc search cache --tags=get,dataset,preprocessed,imagenet,-_NHWC # Tests run-scripts on windows,ubuntu,macos - mlcr get,wkhtmltopdf + + - name: Test native script name variants + run: | + mlcr get,wkhtmltopdf --quiet - name: Test versions continue-on-error: true diff --git a/automation/script/module.py b/automation/script/module.py index 731d3b075..641382707 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -502,8 +502,7 @@ def _run(self, i): ii = {} ii['tags'] = tags_string ii['out'] = None - for key in ["automation", "parsed_automation", - "artifact", "parsed_artifact"]: + for key in ["automation", "artifact", "item", "details"]: if i.get(key): ii[key] = i[key] @@ -2792,13 +2791,19 @@ def search(self, i): # Find MLC script(s) based on thier tags to get their meta (can be more than 1) # Then check if variations exists inside meta - i['tags'] = ','.join(script_tags) + ii = {} + ii['tags'] = ','.join(script_tags) - i['out'] = None - i['common'] = True + ii['out'] = None + ii['common'] = True + + ii['target'] = "script" + for key in ["automation", "item", + "artifact", "details"]: + if i.get(key): + ii[key] = i[key] - i['target_name'] = "script" - r = super(ScriptAutomation, self).search(i) + r = super(ScriptAutomation, self).search(ii) if r['return'] > 0: return r @@ -2901,7 +2906,6 @@ def test(self, i): # Find script item(s) i['out'] = None r = self.search(i) - if r['return'] > 0: return r @@ -2910,7 +2914,6 @@ def test(self, i): for script_item in lst: path = script_item.path meta = script_item.meta - original_meta = script_item.original_meta alias = meta.get('alias', '') uid = meta.get('uid', '') @@ -2970,14 +2973,14 @@ def test(self, i): if given_variations: v_split = [] run_variations = [] - for i, v in enumerate(given_variations): + for v in given_variations: v_split = v.split(",") - for t in v_split: + for index, t in enumerate(v_split): if not t.startswith("_"): # variations must begin with _. We # support both with and without _ # in the meta - given_variations[i] = f"_{t}" + v_split[index] = f"_{t}" if v_split: run_variations.append( ",".join(v_split)) @@ -3018,7 +3021,8 @@ def test(self, i): r = self.action_object.access(ii) if r['return'] > 0: return r - + if is_true(i.get('docker_prune', '')): + docker_prune() return {'return': 0, 'list': lst} ############################################################ @@ -3097,250 +3101,8 @@ def native_run(self, i): return {'return': 0, 'return_code': rc} - ############################################################ - def add(self, i): - """ - Add MLC script - - Args: - (MLC input dict): - - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in MLC CLI or MLC access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (tags) (str): tags to find an MLC script (MLC artifact) - - (script_name) (str): name of script (it will be copied to the new entry and added to the meta) - - (tags) (string or list): tags to be added to meta - - (new_tags) (string or list): new tags to be added to meta (the same as tags) - - (json) (bool): if True, record JSON meta instead of YAML - - (meta) (dict): preloaded meta - - (template) (string): template to use (python) - (python) (bool): template=python - (pytorch) (bool): template=pytorch - ... - - Returns: - (MLC return dict): - - * return (int): return code == 0 if no error and >0 if error - * (error) (str): error string if return>0 - - """ - - import shutil - - console = i.get('out') == 'con' - logger = self.action_object.logger - - # Try to find script artifact by alias and/or tags - # ii = utils.sub_input(i, self.cmind.cfg['artifact_keys']) - ii = {} - ii['tags'] = tags_string - ii['out'] = None - - for key in ["automation", "parsed_automation", - "artifact", "parsed_artifact"]: - if i.get(key): - ii[key] = i[key] - - parsed_artifact = i.get('parsed_artifact', []) - - artifact_obj = parsed_artifact[0] if len(parsed_artifact) > 0 else None - artifact_repo = parsed_artifact[1] if len( - parsed_artifact) > 1 else None - - script_name = '' - if 'script_name' in i: - script_name = i.get('script_name', '').strip() - del (i['script_name']) - - if script_name != '' and not os.path.isfile(script_name): - return {'return': 1, - 'error': 'file {} not found'.format(script_name)} - - # Move tags from input to meta of the newly created script artifact - res = utils.convert_tags_to_list(i['tags']) - if res['return'] > 0: - return res - tags_list = res['tags'] - if 'tags' in i: - del (i['tags']) - - if len(tags_list) == 0: - if console: - x = input( - 'Please specify a combination of unique tags separated by comma for this script: ') - x = x.strip() - if x != '': - tags_list = x.split(',') - - if len(tags_list) == 0: - return { - 'return': 1, 'error': 'you must specify a combination of unique tags separate by comman using "--new_tags"'} - - # Add placeholder (use common action) - ii['out'] = 'con' - # Avoid recursion - use internal MLC add function to add the script - # artifact - ii['common'] = True - - # Check template path - template_dir = 'template' - - template = i.get('template', '') - - if template == '': - if i.get('python', False): - template = 'python' - elif i.get('pytorch', False): - template = 'pytorch' - - if template != '': - template_dir += '-' + template - - template_path = os.path.join(self.path, template_dir) - - if not os.path.isdir(template_path): - return {'return': 1, 'error': 'template path {} not found'.format( - template_path)} - - # Check if preloaded meta exists - meta = { - 'cache': False - # 20240127: Grigori commented that because newly created script meta looks ugly - # 'new_env_keys':[], - # 'new_state_keys':[], - # 'input_mapping':{}, - # 'docker_input_mapping':{}, - # 'deps':[], - # 'prehook_deps':[], - # 'posthook_deps':[], - # 'post_deps':[], - # 'versions':{}, - # 'variations':{}, - # 'input_description':{} - } - - fmeta = os.path.join( - template_path, - self.action_object.cfg['file_cmeta']) - - r = utils.load_yaml_and_json(fmeta) - if r['return'] == 0: - utils.merge_dicts({'dict1': meta, - 'dict2': r['meta'], - 'append_lists': True, - 'append_unique': True}) - - # Check meta from CMD - xmeta = i.get('meta', {}) - - if len(xmeta) > 0: - utils.merge_dicts({'dict1': meta, 'dict2': xmeta, - 'append_lists': True, 'append_unique': True}) - - meta['automation_alias'] = self.meta['alias'] - meta['automation_uid'] = self.meta['uid'] - meta['tags'] = tags_list - - script_name_base = script_name - script_name_ext = '' - if script_name != '': - # separate name and extension - j = script_name.rfind('.') - if j >= 0: - script_name_base = script_name[:j] - script_name_ext = script_name[j:] - - meta['script_name'] = script_name_base - - ii['meta'] = meta - ii['action'] = 'add' - - use_yaml = True if not i.get('json', False) else False - - if use_yaml: - ii['yaml'] = True - - ii['automation'] = 'script,5b4e0237da074764' - - for k in ['parsed_automation', 'parsed_artifact']: - if k in ii: - del ii[k] - - if artifact_repo is not None: - ii['artifact'] = utils.assemble_object2( - artifact_repo) + ':' + utils.assemble_object2(artifact_obj) - - r_obj = self.action_object.access(ii) - if r_obj['return'] > 0: - return r_obj - - new_script_path = r_obj['path'] - - if console: - logger.info('Created script in {}'.format(new_script_path)) - - # Copy files from template (only if exist) - files = [ - (template_path, 'README-extra.md', ''), - (template_path, 'customize.py', ''), - (template_path, 'main.py', ''), - (template_path, 'requirements.txt', ''), - (template_path, 'install_deps.bat', ''), - (template_path, 'install_deps.sh', ''), - (template_path, 'plot.bat', ''), - (template_path, 'plot.sh', ''), - (template_path, 'analyze.bat', ''), - (template_path, 'analyze.sh', ''), - (template_path, 'validate.bat', ''), - (template_path, 'validate.sh', '') - ] - - if script_name == '': - files += [(template_path, 'run.bat', ''), - (template_path, 'run.sh', '')] - else: - if script_name_ext == '.bat': - files += [(template_path, 'run.sh', script_name_base + '.sh')] - files += [('', script_name, script_name)] - - else: - files += [(template_path, 'run.bat', - script_name_base + '.bat')] - files += [('', script_name, script_name_base + '.sh')] - - for x in files: - path = x[0] - f1 = x[1] - f2 = x[2] - - if f2 == '': - f2 = f1 - - if path != '': - f1 = os.path.join(path, f1) - - if os.path.isfile(f1): - f2 = os.path.join(new_script_path, f2) - - if console: - logger.info(' * Copying {} to {}'.format(f1, f2)) - - shutil.copyfile(f1, f2) - - return r_obj - ########################################################################## + def _get_name_for_dynamic_variation_tag(script, variation_tag): ''' Returns the variation name in meta for the dynamic_variation_tag @@ -5101,6 +4863,17 @@ def _update_env(env, key=None, value=None): return {'return': 0} +def docker_prune(): + try: + # Run the docker prune command with -a (removes all unused images, not + # just dangling ones) + result = subprocess.run(["docker", "system", "prune", "-a", "-f"], + capture_output=True, text=True, check=True) + print("Docker prune output:\n", result.stdout) + except subprocess.CalledProcessError as e: + print("Error while running Docker prune:\n", e.stderr) + + ########################################################################## def update_env_with_values(env, fail_on_not_found=False, extra_env=None): """ diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index a3faa4254..41ed35745 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -42,12 +42,14 @@ tests: fail_safe: 'yes' ignore_missing: 'yes' test-all-variations: 'yes' + docker_prune: 'yes' - docker: 'yes' docker_os: ubuntu docker_os_version: '24.04' fail_safe: 'yes' ignore_missing: 'yes' test-all-variations: 'yes' + docker_prune: 'yes' uid: bb0393afa8404a11 variations: cmake: @@ -693,7 +695,7 @@ variations: unzip: env: MLC_SYS_UTIL_NAME: unzip - MLC_SYS_UTIL_VERSION_CMD: unzip --version + MLC_SYS_UTIL_VERSION_CMD: unzip -v new_env_keys: - MLC_UNZIP_VERSION state: diff --git a/script/install-tensorflow-from-src/meta.yaml b/script/install-tensorflow-from-src/meta.yaml index c219d1179..5697128d3 100644 --- a/script/install-tensorflow-from-src/meta.yaml +++ b/script/install-tensorflow-from-src/meta.yaml @@ -56,7 +56,7 @@ versions: version_max_usable: 17.0.6 version_min: 17.0.6 - tags: get,bazel - version: 6.5.0 + version: 7.4.1 env: MLC_GIT_CHECKOUT: master v1.15.0: diff --git a/script/run-mlperf-inference-mobilenet-models/meta.yaml b/script/run-mlperf-inference-mobilenet-models/meta.yaml index 813b1ef93..2eb6647b3 100644 --- a/script/run-mlperf-inference-mobilenet-models/meta.yaml +++ b/script/run-mlperf-inference-mobilenet-models/meta.yaml @@ -159,13 +159,15 @@ variations: group: optimization use-neon: alias: neon + tests: run_inputs: - env: MLC_TEST_ONE_RUN: 'yes' - variations_list: - - tflite - - accuracy_only + variations_list: + - tflite,find-performance adr: compiler: tags: gcc + imagenet-preprocessed: + tags: _500 diff --git a/script/template-script/COPYRIGHT.md b/script/template-script/COPYRIGHT.md deleted file mode 100644 index 9e44ad290..000000000 --- a/script/template-script/COPYRIGHT.md +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Notice - -© 2022-2025 MLCommons. All Rights Reserved. - -This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: - -[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) - -Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. From 7e54b55f602e6f00c697731c3cf970f103ddbf0e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 9 Mar 2025 21:32:34 +0000 Subject: [PATCH 20/27] Added libgl deps for imagenet preprocessing (#304) * Added libgl deps for imagenet preprocessing --- .../meta.yaml | 2 ++ script/get-generic-sys-util/customize.py | 2 +- script/get-generic-sys-util/meta.yaml | 15 +++++++++++++++ .../get-preprocessed-dataset-imagenet/meta.yaml | 1 + script/get-sys-utils-cm/run-ubuntu.sh | 1 - 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 720d59514..fe7793e2d 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1252,6 +1252,7 @@ variations: group: models deps: - tags: get,generic-python-lib,_opencv-python + - tags: get,generic-sys-util,_libgl - tags: get,generic-python-lib,_numpy names: - numpy @@ -1270,6 +1271,7 @@ variations: MLC_MLPERF_USE_MLCOMMONS_RUN_SCRIPT: "yes" deps: - tags: get,generic-python-lib,_opencv-python + - tags: get,generic-sys-util,_libgl - tags: get,generic-python-lib,_numpy names: - numpy diff --git a/script/get-generic-sys-util/customize.py b/script/get-generic-sys-util/customize.py index 74b8c75b6..0efd666bf 100644 --- a/script/get-generic-sys-util/customize.py +++ b/script/get-generic-sys-util/customize.py @@ -11,7 +11,7 @@ def preprocess(i): state = i['state'] automation = i['automation'] - # Use VERSION_CMD and CHECK_CMD if no CHECK_CMD is set + # Use VERSION_CMD as CHECK_CMD if no CHECK_CMD is set if env.get('MLC_SYS_UTIL_VERSION_CMD', '') != '' and env.get( 'MLC_SYS_UTIL_CHECK_CMD', '') == '': env['MLC_SYS_UTIL_CHECK_CMD'] = env['MLC_SYS_UTIL_VERSION_CMD'] diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index 41ed35745..cc278c21f 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -110,6 +110,21 @@ variations: brew: '' dnf: '' yum: '' + + libgl: + env: + MLC_SYS_UTIL_NAME: libgl # tbd: regular expression for version as well as whether its installed? + MLC_SYS_UTIL_CHECK_CMD: 'ldconfig -p | grep -i libgl.so.*' + default_env: + MLC_GENERIC_SYS_UTIL_IGNORE_MISSING_PACKAGE: yes + new_env_keys: + - MLC_LIBGL_VERSION + state: + libgl: # tbd: complete for other flavours of linux + apt: libgl1 + brew: '' + dnf: mesa-libGL + yum: mesa-libGL libsm6: env: MLC_SYS_UTIL_NAME: libsm6 # tbd: regular expression for version as well as whether its installed? diff --git a/script/get-preprocessed-dataset-imagenet/meta.yaml b/script/get-preprocessed-dataset-imagenet/meta.yaml index eea817b2f..124f73572 100644 --- a/script/get-preprocessed-dataset-imagenet/meta.yaml +++ b/script/get-preprocessed-dataset-imagenet/meta.yaml @@ -51,6 +51,7 @@ deps: MLC_IMAGENET_PREPROCESSED_PATH: - 'on' tags: mlperf,mlcommons,inference,source,src +- tags: get,generic-sys-util,_libgl docker: run: false env: diff --git a/script/get-sys-utils-cm/run-ubuntu.sh b/script/get-sys-utils-cm/run-ubuntu.sh index df0328c40..df10e4ef3 100644 --- a/script/get-sys-utils-cm/run-ubuntu.sh +++ b/script/get-sys-utils-cm/run-ubuntu.sh @@ -52,7 +52,6 @@ ${MLC_SUDO} ${MLC_APT_TOOL} update && \ libgl1 \ libjpeg9-dev \ unzip \ - libgl1 \ zlib1g-dev # Install Python deps though preference is to install them From e023a0f389f0ee82136e14c6c621462f595abbef Mon Sep 17 00:00:00 2001 From: Arjun Date: Tue, 11 Mar 2025 21:18:59 +0530 Subject: [PATCH 21/27] Fix update of dynamic values from preprocess --- automation/script/module.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automation/script/module.py b/automation/script/module.py index 641382707..ca5df1e76 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1830,6 +1830,10 @@ def _run(self, i): import json logger.debug(json.dumps(env, indent=2, sort_keys=True)) + r = update_env_with_values(env) + if r['return'] > 0: + return r + # Check chain of pre hook dependencies on other MLC scripts if len(prehook_deps) > 0: logger.debug( From f64752f405dbcd958431a2fc6860f52304635b61 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 12 Mar 2025 17:01:16 +0530 Subject: [PATCH 22/27] Support dirpath for download and extract scripts --- script/download-file/customize.py | 2 ++ script/extract-file/customize.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 64066122f..f525c440f 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -92,6 +92,8 @@ def preprocess(i): if env.get('MLC_DOWNLOAD_PATH', '') != '': download_path = env['MLC_DOWNLOAD_PATH'] + if os.path.isfile(download_path): + download_path = os.path.dirname(download_path) if not os.path.exists(download_path): os.makedirs(download_path, exist_ok=True) os.chdir(download_path) diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index fd6ab8235..d55f8787c 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -37,6 +37,8 @@ def preprocess(i): # later if cache is cleaned) extract_path = env.get('MLC_EXTRACT_PATH', '') if extract_path != '': + if os.path.isfile(extract_path): + extract_path = os.path.dirname(extract_path) if not os.path.exists(extract_path): os.makedirs(extract_path, exist_ok=True) From ed55baa66b057a5f11dcf5bfbe6b89671dc2f84f Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 15 Mar 2025 18:40:57 +0530 Subject: [PATCH 23/27] Fixes for llvm install, added test --- automation/script/module.py | 5 ----- script/install-llvm-src/customize.py | 17 +++++++++-------- script/install-llvm-src/meta.yaml | 16 ++++++++++++++-- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index ca5df1e76..36752d301 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4789,11 +4789,6 @@ def enable_or_skip_script(meta, env): (AND function) """ - if not isinstance(meta, dict): - logger.warn( - "The meta entry is not a dictionary for skip/enable if_env: %s", - meta) - for key in meta: meta_key = [str(v).lower() for v in meta[key]] if key in env: diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 843e2027d..b1f5f303b 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -11,6 +11,8 @@ def preprocess(i): env = i['env'] + q = '"' if os_info['platform'] == 'windows' else "'" + clang_file_name = "clang" extra_cmake_options = '' @@ -27,22 +29,21 @@ def preprocess(i): # env['USE_LLVM'] = install_prefix # env['LLVM_DIR'] = os.path.join(env['USE_LLVM'], "lib", "cmake", "llvm") else: - if env.get('MLC_LLVM_ENABLE_RUNTIMES', '') != '': - enable_runtimes = env['MLC_LLVM_ENABLE_RUNTIMES'].replace(":", ";") + if env.get('+MLC_LLVM_ENABLE_RUNTIMES', '') != '': + enable_runtimes = ";".join(env['+MLC_LLVM_ENABLE_RUNTIMES']) else: enable_runtimes = '' - if env.get('MLC_LLVM_ENABLE_PROJECTS', '') != '': - enable_projects = env['MLC_LLVM_ENABLE_PROJECTS'].replace(":", ";") + if env.get('+MLC_LLVM_ENABLE_PROJECTS', '') != '': + enable_projects = ";".join(env['+MLC_LLVM_ENABLE_PROJECTS']) else: enable_projects = '' llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] - cmake_cmd = "cmake " + os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm") + " -GNinja -DCMAKE_BUILD_TYPE=" + llvm_build_type + " -DLLVM_ENABLE_PROJECTS=" + enable_projects + " -DLLVM_ENABLE_RUNTIMES='" + \ - enable_runtimes + "' -DCMAKE_INSTALL_PREFIX=" + install_prefix + \ - " -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD=X86 " + \ - extra_cmake_options + targets_to_buuild = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') + + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index f1dd3657d..1973a78c0 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -62,8 +62,17 @@ variations: clang: default: true env: - MLC_LLVM_ENABLE_PROJECTS: clang + +MLC_LLVM_ENABLE_PROJECTS: clang group: clang + lld: + default: true + env: + +MLC_LLVM_ENABLE_PROJECTS: lld + group: lld + no-clang: + group: clang + no-lld: + group: lld debug: env: MLC_LLVM_BUILD_TYPE: debug @@ -198,7 +207,7 @@ variations: group: repo runtimes.#: env: - MLC_LLVM_ENABLE_RUNTIMES: '#' + +MLC_LLVM_ENABLE_RUNTIMES: '#' sha.#: env: MLC_GIT_CHECKOUT_SHA: '#' @@ -208,3 +217,6 @@ variations: env: MLC_GIT_CHECKOUT_TAG: '#' versions: {} +tests: + run_inputs: + - version: "20.1.0" From c042743329154c6af86ed1915dfbe57abfd7702d Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery <82945126+sujik18@users.noreply.github.com> Date: Sat, 15 Mar 2025 19:02:13 +0530 Subject: [PATCH 24/27] Fix for ResNet50 Closed Division GitHub Action (#307) * Fix for test-mlpref-resnet50-closed-division * Update test-mlperf-inference-resnet50-closed-division.yml --------- Co-authored-by: Arjun Suresh --- ...erf-inference-resnet50-closed-division.yml | 55 +++---------------- 1 file changed, 8 insertions(+), 47 deletions(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 574d0ae19..586317439 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -1,18 +1,9 @@ name: MLPerf inference ResNet50 Closed Division for testing Compliance on: - # push: - # branches: - # - test schedule: - - cron: '0 0 * * 0' # Runs once a week - pull_request_target: - branches: [ "main", "dev" ] - paths: - - '.github/workflows/test-mlperf-inference-resnet50-closed-division.yml' - - '**' - - '!**.md' - # workflow_dispatch: # Allows manual triggering of the workflow + - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC + workflow_dispatch: {} # Allows manual triggering of the workflow jobs: build: name: MLPerf inference MLCommons ResNet50 Closed Division @@ -54,48 +45,18 @@ jobs: - name: Pull MLOps repo run: | - mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - + mlc pull repo mlcommons@mlperf-automations --branch=dev - - name: Test MLPerf Inference ResNet50 Offline (Windows) + - name: Test MLPerf Inference ResNet50 (Windows) if: matrix.os == 'windows-latest' run: | - mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - - name: Test MLPerf Inference ResNet50 Singlestream (Windows) - if: matrix.os == 'windows-latest' - run: | - mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --target_latency=10 - - - name: Test MLPerf Inference ResNet50 Multistream (Windows) - if: matrix.os == 'windows-latest' - run: | - mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream , --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 - - - name: Compliance Test MLPerf Inference ResNet50 (Windows) - if: matrix.os == 'windows-latest' - run: mlcr run-mlperf, inference, _submission, _short,_all-scenarios, --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid --compliance=yes # mentioning only --compliance currently doesnt work needs to be --compliance=yes - + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + - name: Test MLPerf Inference ResNet50 Offline(Linux/macOS) if: matrix.os != 'windows-latest' run: | - mlcr run-mlperf, inference, _submission, _short, scenario=Offline, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid - - - name: Test MLPerf Inference ResNet50 SingleStream(Linux/macOS) - if: matrix.os != 'windows-latest' - run: | - mlcr run-mlperf, inference, _submission, _short, scenario=SingleStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 - - - name: Test MLPerf Inference ResNet50 MultiStream(Linux/macOS) - if: matrix.os != 'windows-latest' - run: | - mlcr run-mlperf, inference, _submission, _short, scenario=MultiStream, --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --quiet --execution_mode=valid --target_latency=10 - - - name: Compliance Test MLPerf Inference ResNet50 (Linux/macOS) - if: matrix.os != 'windows-latest' - run: | - mlcr run-mlperf, inference, _submission, _short, _all-scenarios --division=closed, --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 -v --quiet --execution_mode=valid --compliance=yes - + mlcr run-mlperf,inference,_submission,_short,_all-scenarios --division=closed --submitter="MLCommons" --pull_changes=yes --pull_inference_changes=yes --hw_name="gh_${{ matrix.os }} x86" --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --test_query_count=1000 --quiet --execution_mode=valid + # Step for Linux/MacOS - name: Randomly Execute Step (Linux/MacOS) if: runner.os != 'Windows' From 672088926f52978b5c8badd2089be45369077a76 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 15 Mar 2025 18:41:53 +0000 Subject: [PATCH 25/27] Fix lists in install-llvm-src (#309) * Fix lists in install-llvm-src * Support lists in env update in script automation --- .github/workflows/build_wheel.yml | 1 - automation/script/module.py | 30 +++++++++++++++------------- script/install-llvm-src/customize.py | 4 ++-- script/install-llvm-src/meta.yaml | 9 ++++++--- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 8de31237d..1edf42589 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -10,7 +10,6 @@ on: paths: - VERSION - jobs: build_wheels: diff --git a/automation/script/module.py b/automation/script/module.py index 36752d301..840b69554 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -460,7 +460,11 @@ def _run(self, i): # Add permanent env from OS (such as MLC_WINDOWS:"yes" on Windows) env_from_os_info = os_info.get('env', {}) if len(env_from_os_info) > 0: - env.update(env_from_os_info) + # env.update(env_from_os_info) + utils.merge_dicts({'dict1': env, + 'dict2': env_from_os_info, + 'append_lists': True, + 'append_unique': True}) # take some env from the user environment keys = [ @@ -833,7 +837,10 @@ def _run(self, i): script_item_env = meta.get('env', {}) # print(f"script meta env= {script_item_env}") - env.update(script_item_env) + utils.merge_dicts({'dict1': env, + 'dict2': script_item_env, + 'append_lists': True, + 'append_unique': True}) # print(f"env = {env}") script_item_state = meta.get('state', {}) @@ -1915,8 +1922,7 @@ def _run(self, i): # Finalize script # Force consts in the final new env and state - utils.merge_dicts({'dict1': env, 'dict2': const, - 'append_lists': True, 'append_unique': True}) + env.update(const) utils.merge_dicts({'dict1': state, 'dict2': const_state, 'append_lists': True, @@ -3473,7 +3479,7 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a if r['return'] > 0: return r - # Update env/state with cost + # Update env/state with const env.update(const) utils.merge_dicts({'dict1': state, 'dict2': const_state, @@ -5052,10 +5058,7 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): 'return': 16, 'error': 'script {} not found - please add one'.format(path_to_run_script)} # Update env and state with const - utils.merge_dicts({'dict1': env, - 'dict2': const, - 'append_lists': True, - 'append_unique': True}) + env.update(const) utils.merge_dicts({'dict1': state, 'dict2': const_state, 'append_lists': True, 'append_unique': True}) @@ -5281,8 +5284,7 @@ def run_detect_version(customize_code, customize_common_input, logger.debug(recursion_spaces + ' - Running detect_version ...') # Update env and state with const - utils.merge_dicts({'dict1': env, 'dict2': const, - 'append_lists': True, 'append_unique': True}) + env.update(const) utils.merge_dicts({'dict1': state, 'dict2': const_state, 'append_lists': True, @@ -5312,8 +5314,7 @@ def run_postprocess(customize_code, customize_common_input, recursion_spaces, logger.debug(recursion_spaces + ' - Running postprocess ...') # Update env and state with const - utils.merge_dicts({'dict1': env, 'dict2': const, - 'append_lists': True, 'append_unique': True}) + env.update(const) utils.merge_dicts({'dict1': state, 'dict2': const_state, 'append_lists': True, @@ -5745,7 +5746,8 @@ def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps env.setdefault(key, default_env[key]) update_env = meta.get('env', {}) - env.update(update_env) + utils.merge_dicts({'dict1': env, 'dict2': update_env, + 'append_lists': True, 'append_unique': True}) update_meta_if_env = meta.get('update_meta_if_env', []) update_meta_if_env_from_state = run_state.get('update_meta_if_env', []) diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index b1f5f303b..ea6b02c54 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -41,9 +41,9 @@ def preprocess(i): llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] - targets_to_buuild = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') + targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') - cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 1973a78c0..0c8d91c43 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -62,12 +62,14 @@ variations: clang: default: true env: - +MLC_LLVM_ENABLE_PROJECTS: clang + +MLC_LLVM_ENABLE_PROJECTS: + - clang group: clang lld: default: true env: - +MLC_LLVM_ENABLE_PROJECTS: lld + +MLC_LLVM_ENABLE_PROJECTS: + - lld group: lld no-clang: group: clang @@ -207,7 +209,8 @@ variations: group: repo runtimes.#: env: - +MLC_LLVM_ENABLE_RUNTIMES: '#' + +MLC_LLVM_ENABLE_RUNTIMES: + - '#' sha.#: env: MLC_GIT_CHECKOUT_SHA: '#' From f2fbb6c10406264fe659519b6c75c9a257a36ade Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 16 Mar 2025 21:44:54 +0000 Subject: [PATCH 26/27] Fix wkhtmltopdf installation on windows (#312) * Fix wkhtmltopdf installation on windows * Update README.md --- README.md | 4 ++-- script/get-generic-sys-util/meta.yaml | 1 + script/get-wkhtmltopdf/meta.yaml | 6 ++++-- script/get-wkhtmltopdf/run.bat | 27 --------------------------- 4 files changed, 7 insertions(+), 31 deletions(-) delete mode 100644 script/get-wkhtmltopdf/run.bat diff --git a/README.md b/README.md index fba9394e4..0146c5752 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Welcome to the **MLPerf Automations and Scripts** repository! This repository is your go-to resource for tools, automations, and scripts designed to streamline the execution of **MLPerf benchmarks**—with a strong emphasis on **MLPerf Inference benchmarks**. -Starting **January 2025**, MLPerf automation scripts will be powered by the advanced [MLCFlow](https://github.com/mlcommons/mlcflow) automation interface. This modern framework replaces the previous [Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm), providing a more robust, efficient, and self-contained solution for benchmarking workflows, making MLPerf automations independent of any external projects. +Starting **January 2025**, MLPerf automation scripts is powered by [MLCFlow](https://github.com/mlcommons/mlcflow) automation interface. This new and simplified framework replaces the previous [Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm), providing a more robust, efficient, and self-contained solution for benchmarking workflows, making MLPerf automations independent of any external projects. --- @@ -17,7 +17,6 @@ Starting **January 2025**, MLPerf automation scripts will be powered by the adva - **Automated Benchmarking** – Simplifies running MLPerf Inference benchmarks with minimal manual intervention. - **Modular and Extensible** – Easily extend the scripts to support additional benchmarks and configurations. - **Seamless Integration** – Compatible with Docker, cloud environments, and local machines. -- **MLCFlow (MLC) Integration** – Utilizes the MLC framework to enhance reproducibility and automation. --- @@ -61,6 +60,7 @@ This project is made possible through the generous support of: - [OctoML](https://octoml.ai) - [cKnowledge.org](https://cKnowledge.org) - [cTuning Foundation](https://cTuning.org) +- [GATEOverflow](https://gateoverflow.in) - [MLCommons](https://mlcommons.org) We appreciate their contributions and sponsorship! diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index cc278c21f..bd4006e72 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -760,6 +760,7 @@ variations: wkhtmltopdf: apt: wkhtmltopdf brew: wkhtmltopdf + choco: wkhtmltopdf xfonts-base: env: MLC_SYS_UTIL_NAME: xfonts_base diff --git a/script/get-wkhtmltopdf/meta.yaml b/script/get-wkhtmltopdf/meta.yaml index 3e0ff443a..471c502eb 100755 --- a/script/get-wkhtmltopdf/meta.yaml +++ b/script/get-wkhtmltopdf/meta.yaml @@ -12,9 +12,11 @@ deps: MLC_HOST_OS_TYPE: - windows - tags: get,generic-sys-util,_wkhtmltopdf - enable_if_env: + enable_if_any_env: MLC_INSTALL_FROM_PACKAGE_MANAGER: - - on + - on + MLC_HOST_OS_TYPE: + - windows uid: 67ec874a3dfe4b87 docker: pre_run_cmds: diff --git a/script/get-wkhtmltopdf/run.bat b/script/get-wkhtmltopdf/run.bat deleted file mode 100644 index 6ccb6b19a..000000000 --- a/script/get-wkhtmltopdf/run.bat +++ /dev/null @@ -1,27 +0,0 @@ -:: Define download URL and filename -set "URL=https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox-0.12.6-1.msvc2015-win64.exe" -set "FILE=wkhtmltox-0.12.6-1.msvc2015-win64.exe" - -:: Download the installer if it doesn't already exist -if not exist "%FILE%" ( - echo Downloading %FILE%... - powershell -Command "(New-Object System.Net.WebClient).DownloadFile('%URL%', '%CD%\\%FILE%')" - if %ERRORLEVEL% neq 0 ( - echo Download failed! - exit /b %ERRORLEVEL% - ) -) - -:: Install the software (silent mode) -echo Installing wkhtmltopdf... -start /wait %FILE% /quiet - -if %ERRORLEVEL% neq 0 ( - echo Installation failed! - exit /b %ERRORLEVEL% -) - -echo Installation successful! - -endlocal -exit /b 0 From 88110943b3ab266c0bc901c8ca08b6248fada9e1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 17 Mar 2025 01:12:49 +0000 Subject: [PATCH 27/27] Improve get-rclone-config (#313) * Added support for config-name.# in get-rclone-config * Support write rclone config --- automation/script/module.py | 1 + script/get-rclone-config/meta.yaml | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 840b69554..db32015b4 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2593,6 +2593,7 @@ def _update_variation_tags_from_variations( variation_name = self._get_name_for_dynamic_variation_tag( variation_name) + # TODO: Move this to a function and apply it for combination of variations too # base variations are automatically turned on. Only # variations outside of any variation group can be added as # a base_variation diff --git a/script/get-rclone-config/meta.yaml b/script/get-rclone-config/meta.yaml index f2f3332f0..a7bd8e5b3 100644 --- a/script/get-rclone-config/meta.yaml +++ b/script/get-rclone-config/meta.yaml @@ -8,18 +8,31 @@ tags: - rclone-config uid: 6c59ddbc6cd046e3 variations: + read-only: + group: permission + default: true + env: + MLC_RCLONE_DRIVE_SCOPE: drive.readonly + write: + group: permission + env: + MLC_RCLONE_DRIVE_SCOPE: drive mlc-inference: env: MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com' mlperf-llama2: env: - MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama2 drive config_is_local=false scope=drive.readonly root_folder_id=11tBZvvrh0FCm3XuR5E849K42TqftYdUF' + MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama2 drive config_is_local=false scope=<<>> root_folder_id=11tBZvvrh0FCm3XuR5E849K42TqftYdUF' MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-llama2:' mlperf-llama3-1: env: - MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama3-1 drive config_is_local=false scope=drive.readonly root_folder_id=12K-2yvmr1ZSZ7SLrhidCbWc0BriN98am' + MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-llama3-1 drive config_is_local=false scope=<<>> root_folder_id=12K-2yvmr1ZSZ7SLrhidCbWc0BriN98am' MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-llama3-1:' waymo: env: - MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-waymo drive config_is_local=false scope=drive.readonly root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5' + MLC_RCLONE_CONFIG_CMD: 'rclone config create mlc-waymo drive config_is_local=false scope=<<>> root_folder_id=1xbfnaUurFeXliFFl1i1gj48eRU2NDiH5' MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect mlc-waymo:' + config-name.#: + env: + MLC_RCLONE_CONFIG_CMD: 'rclone config create # drive config_is_local=false scope=<<>> root_folder_id=<<>>' + MLC_RCLONE_CONNECT_CMD: 'rclone config reconnect #:'