From db48ff1a7f0cca62b4af09a271791b92c49717b9 Mon Sep 17 00:00:00 2001 From: Arjun Date: Thu, 22 May 2025 20:25:47 +0530 Subject: [PATCH 01/11] Added get-lib-jemalloc --- script/get-lib-jemalloc/customize.py | 29 +++++++++++++++++++ script/get-lib-jemalloc/meta.yaml | 43 ++++++++++++++++++++++++++++ script/get-lib-jemalloc/run.sh | 13 +++++++++ 3 files changed, 85 insertions(+) create mode 100644 script/get-lib-jemalloc/customize.py create mode 100644 script/get-lib-jemalloc/meta.yaml create mode 100644 script/get-lib-jemalloc/run.sh diff --git a/script/get-lib-jemalloc/customize.py b/script/get-lib-jemalloc/customize.py new file mode 100644 index 000000000..a82d6a829 --- /dev/null +++ b/script/get-lib-jemalloc/customize.py @@ -0,0 +1,29 @@ +from mlc import utils +import os +import subprocess + + +def preprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + lib_path = os.path.join(os.getcwd(), "obj", "lib") + + env['+LD_LIBRARY_PATH'] = lib_path + env['MLC_JEMALLOC_LIB_PATH'] = lib_path + env['MLC_DEPENDENT_CACHED_PATH'] = os.path.join(lib_path, "libjemalloc.so") + + return {'return': 0} diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml new file mode 100644 index 000000000..b3e30d2c6 --- /dev/null +++ b/script/get-lib-jemalloc/meta.yaml @@ -0,0 +1,43 @@ +alias: get-lib-jemalloc +automation_alias: script +automation_uid: 5b4e0237da074764 +category: MLC Script Template +deps: + - tags: get,git,repo + env: + MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_JEMALLOC_SRC_PATH + force_env_keys: + - MLC_GIT_* + update_tags_from_env_with_prefix: + _branch.: + - MLC_GIT_CHECKOUT + _repo.: + - MLC_GIT_URL + _sha.: + - MLC_GIT_SHA + _submodules.: + - MLC_GIT_SUBMODULES + names: + - jemalloc-repo + extra_cache_tags: jemalloc,repo,jemalloc-repo +new_env_keys: + - MLC_JEMALLOC_LIB_PATH + - +LD_LIBRARY_PATH +new_state_keys: [] +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- get +- lib +- lib-jemalloc +- jemalloc +tests: + run_inputs: [] +uid: 406439a446e04fb7 +variations: + version.official: + group: version + default: true + env: + MLC_GIT_URL: https://github.com/jemalloc/jemalloc.git diff --git a/script/get-lib-jemalloc/run.sh b/script/get-lib-jemalloc/run.sh new file mode 100644 index 000000000..2d156cf42 --- /dev/null +++ b/script/get-lib-jemalloc/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +#Add your run commands here... +# run "$MLC_RUN_CMD" +cd ${MLC_JEMALLOC_SRC_PATH} +autoconf +cd - +mkdir -p obj +cd obj +${MLC_JEMALLOC_SRC_PATH}/configure --enable-autogen +make From 3cbb58e8900a55ee5bc924d36dc4da852eae4f70 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 17:24:13 +0100 Subject: [PATCH 02/11] Update meta.yaml --- script/get-lib-jemalloc/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml index b3e30d2c6..93221732c 100644 --- a/script/get-lib-jemalloc/meta.yaml +++ b/script/get-lib-jemalloc/meta.yaml @@ -33,7 +33,8 @@ tags: - lib-jemalloc - jemalloc tests: - run_inputs: [] + run_inputs: + - quiet: true uid: 406439a446e04fb7 variations: version.official: From c3bf3f78cff87388ecdafdd0b638a42d311a6c17 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 17:36:33 +0100 Subject: [PATCH 03/11] Update run-tests-on-modified-meta.yml --- .../workflows/run-tests-on-modified-meta.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 244b18a6f..adfd76453 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -17,7 +17,7 @@ jobs: - name: 'Checkout' uses: actions/checkout@v4 with: - fetch-depth: 2 + fetch-depth: 0 - name: Setup Python uses: actions/setup-python@v2 @@ -28,13 +28,20 @@ jobs: run: | pip install pyyaml - - name: Get changed files + - name: Fetch base branch + run: | + git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} + + - name: Get list of changed files id: modified-files run: | - git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} - git fetch upstream - changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only) - echo "$changed_files" | python3 .github/scripts/list_modified_files.py + git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt + echo "::set-output name=files::$(paste -sd, changed_files.txt)" + + - name: Filter changed files + id: filter-modified-files + run: | + echo "${{ steps.modified-files.outputs.files }}" | python3 .github/scripts/list_modified_files.py process_modified_files: runs-on: ubuntu-latest From 1bfdaccb7439cbd993fb6d55094cc7449d04ff99 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 17:47:36 +0100 Subject: [PATCH 04/11] Update run-tests-on-modified-meta.yml --- .../workflows/run-tests-on-modified-meta.yml | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index adfd76453..f642a3e11 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -1,4 +1,3 @@ -# This workflow will run configured tests for any updated MLC script name: Test script on modified meta on: @@ -10,43 +9,57 @@ on: jobs: get_modified_files: runs-on: ubuntu-latest + + # expose the final processed_files JSON to downstream jobs outputs: - processed_files: ${{ steps.modified-files.outputs.processed_files }} + processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} steps: - - name: 'Checkout' - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' - - name: Install dependencies - run: | - pip install pyyaml + - name: Install dependencies + run: | + pip install pyyaml - - name: Fetch base branch - run: | + - name: Fetch base branch + run: | git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - name: Get list of changed files - id: modified-files - run: | + - name: Get list of changed files + id: modified-files + run: | git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt - echo "::set-output name=files::$(paste -sd, changed_files.txt)" + files=$(paste -sd, changed_files.txt) + # set as a step output + echo "files=$files" >> $GITHUB_OUTPUT - - name: Filter changed files - id: filter-modified-files - run: | - echo "${{ steps.modified-files.outputs.files }}" | python3 .github/scripts/list_modified_files.py + - name: Filter changed files + id: filter-modified-files + # pull in the raw file list via env + env: + FILES: ${{ steps.modified-files.outputs.files }} + run: | + # list_modified_files.py should emit a JSON array on stdout, + # e.g. '[{"file":"script/foo/meta.yaml","uid":"foo","num_run":1}, …]' + processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) + # expose that JSON for the job output + echo "processed_files=$processed" >> $GITHUB_OUTPUT + echo $processed process_modified_files: - runs-on: ubuntu-latest needs: get_modified_files - if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != '' + runs-on: ubuntu-latest + # only run if there is at least one file to process + if: ${{ needs.get_modified_files.outputs.processed_files != '[]' && needs.get_modified_files.outputs.processed_files != '' }} + strategy: fail-fast: false matrix: @@ -60,8 +73,7 @@ jobs: - name: Process meta.yaml file run: | - echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}" - + echo "Processing ${{ matrix.file_info.file }} (run #${{ matrix.file_info.num_run }})" pip install mlcflow mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet + mlc test script ${{ matrix.file_info.uid }} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet From c0c33df4d335c6433de04300bef04bbbb2d68a0a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 17:53:15 +0100 Subject: [PATCH 05/11] Update run-tests-on-modified-meta.yml --- .github/workflows/run-tests-on-modified-meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index f642a3e11..ff6134512 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -52,7 +52,6 @@ jobs: processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) # expose that JSON for the job output echo "processed_files=$processed" >> $GITHUB_OUTPUT - echo $processed process_modified_files: needs: get_modified_files From a94f106cf7b745bd60be034cf67d44bba8f07878 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 17:59:00 +0100 Subject: [PATCH 06/11] Update run-tests-on-modified-meta.yml --- .../workflows/run-tests-on-modified-meta.yml | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index ff6134512..0ddb8b81f 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -9,60 +9,30 @@ on: jobs: get_modified_files: runs-on: ubuntu-latest - - # expose the final processed_files JSON to downstream jobs outputs: processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - pip install pyyaml - - - name: Fetch base branch - run: | - git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - - name: Get list of changed files - id: modified-files - run: | - git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt - files=$(paste -sd, changed_files.txt) - # set as a step output - echo "files=$files" >> $GITHUB_OUTPUT + # … your checkout, fetch, get-diff, etc … - name: Filter changed files id: filter-modified-files - # pull in the raw file list via env env: FILES: ${{ steps.modified-files.outputs.files }} run: | - # list_modified_files.py should emit a JSON array on stdout, - # e.g. '[{"file":"script/foo/meta.yaml","uid":"foo","num_run":1}, …]' processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) - # expose that JSON for the job output - echo "processed_files=$processed" >> $GITHUB_OUTPUT + processed=${processed:-[]} + echo "processed_files<> $GITHUB_OUTPUT + printf '%s\n' "$processed" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT process_modified_files: needs: get_modified_files runs-on: ubuntu-latest - # only run if there is at least one file to process - if: ${{ needs.get_modified_files.outputs.processed_files != '[]' && needs.get_modified_files.outputs.processed_files != '' }} - + if: needs.get_modified_files.outputs.processed_files != '[]' strategy: fail-fast: false matrix: - file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }} + file_info: ${{ needs.get_modified_files.outputs.processed_files != '' && fromJSON(needs.get_modified_files.outputs.processed_files) || [] }} steps: - name: Checkout repository From abb9f3158ec48c6cc4979ab6aa1d412501c4b6f8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 18:03:01 +0100 Subject: [PATCH 07/11] Update run-tests-on-modified-meta.yml --- .../workflows/run-tests-on-modified-meta.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 0ddb8b81f..cd54163a0 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -12,7 +12,21 @@ jobs: outputs: processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} steps: - # … your checkout, fetch, get-diff, etc … + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch base branch + run: | + git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} + + - name: Get list of changed files + id: modified-files + run: | + git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt + files=$(paste -sd, changed_files.txt) + echo "files=$files" >> $GITHUB_OUTPUT - name: Filter changed files id: filter-modified-files @@ -20,6 +34,7 @@ jobs: FILES: ${{ steps.modified-files.outputs.files }} run: | processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) + # Ensure valid JSON even if empty processed=${processed:-[]} echo "processed_files<> $GITHUB_OUTPUT printf '%s\n' "$processed" >> $GITHUB_OUTPUT @@ -30,9 +45,8 @@ jobs: runs-on: ubuntu-latest if: needs.get_modified_files.outputs.processed_files != '[]' strategy: - fail-fast: false matrix: - file_info: ${{ needs.get_modified_files.outputs.processed_files != '' && fromJSON(needs.get_modified_files.outputs.processed_files) || [] }} + file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }} steps: - name: Checkout repository From f5048669f97cb41934c574d0398cba82966d1244 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 18:21:51 +0100 Subject: [PATCH 08/11] Update run-tests-on-modified-meta.yml --- .../workflows/run-tests-on-modified-meta.yml | 65 ++++++++++++------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 4e6eb0b33..fd3b8792c 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -11,39 +11,54 @@ jobs: runs-on: ubuntu-latest outputs: processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} + steps: - - name: 'Checkout' - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - pip install pyyaml - - - name: Get changed files - id: modified-files - run: | - git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} - git fetch upstream - changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only) - echo "$changed_files" | python3 .github/scripts/list_modified_files.py - - - name: Debug Show processed_files - run: | - echo "Processed files: '${{ steps.filter-modified-files.outputs.processed_files }}'" + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install pyyaml + + - name: Fetch base branch + run: | + git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} + + - name: Get list of changed files + id: modified-files + run: | + git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt + files=$(paste -sd, changed_files.txt) + echo "files=$files" >> $GITHUB_OUTPUT + + - name: Filter changed files + id: filter-modified-files + env: + FILES: ${{ steps.modified-files.outputs.files }} + run: | + processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) + processed=${processed:-[]} + echo "processed_files<> $GITHUB_OUTPUT + echo "$processed" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Debug processed_files output + run: | + echo "Processed files output:" + echo "${{ steps.filter-modified-files.outputs.processed_files }}" process_modified_files: needs: get_modified_files runs-on: ubuntu-latest if: needs.get_modified_files.outputs.processed_files != '[]' strategy: + fail-fast: false matrix: file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files) }} From 4af804f6f10a78a8b28563ec81ff9d1e567d489d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 18:25:40 +0100 Subject: [PATCH 09/11] Update list_modified_files.py --- .github/scripts/list_modified_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/list_modified_files.py b/.github/scripts/list_modified_files.py index c3c2260a7..160c426a3 100644 --- a/.github/scripts/list_modified_files.py +++ b/.github/scripts/list_modified_files.py @@ -34,4 +34,4 @@ def process_files(files): changed_files = sys.stdin.read().strip() processed_files = process_files(changed_files) json_processed_files = json.dumps(processed_files) - print(f"::set-output name=processed_files::{json_processed_files}") + print(json_processed_files) From da0f27ded550164c17aa9ade473ae2ef18b03e33 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 May 2025 18:35:02 +0100 Subject: [PATCH 10/11] Update run-tests-on-modified-meta.yml --- .github/workflows/run-tests-on-modified-meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index fd3b8792c..86932d896 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -43,7 +43,6 @@ jobs: FILES: ${{ steps.modified-files.outputs.files }} run: | processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) - processed=${processed:-[]} echo "processed_files<> $GITHUB_OUTPUT echo "$processed" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT From 5e0b473902e7aa13097a728fc05175714a07b741 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 23 May 2025 02:38:19 +0530 Subject: [PATCH 11/11] Fix github action for mlc tests --- .github/scripts/list_modified_files.py | 2 +- script/get-lib-jemalloc/meta.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/list_modified_files.py b/.github/scripts/list_modified_files.py index 160c426a3..65fdacbd1 100644 --- a/.github/scripts/list_modified_files.py +++ b/.github/scripts/list_modified_files.py @@ -17,7 +17,7 @@ def get_file_info(filepath): def process_files(files): - filenames = files.split() + filenames = files.split(",") return [ { "file": file, diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml index 93221732c..929152152 100644 --- a/script/get-lib-jemalloc/meta.yaml +++ b/script/get-lib-jemalloc/meta.yaml @@ -1,7 +1,8 @@ alias: get-lib-jemalloc automation_alias: script automation_uid: 5b4e0237da074764 -category: MLC Script Template +category: Detection or installation of tools and artifacts +cache: true deps: - tags: get,git,repo env: