diff --git a/.github/scripts/list_modified_files.py b/.github/scripts/list_modified_files.py index c3c2260a7..65fdacbd1 100644 --- a/.github/scripts/list_modified_files.py +++ b/.github/scripts/list_modified_files.py @@ -17,7 +17,7 @@ def get_file_info(filepath): def process_files(files): - filenames = files.split() + filenames = files.split(",") return [ { "file": file, @@ -34,4 +34,4 @@ def process_files(files): changed_files = sys.stdin.read().strip() processed_files = process_files(changed_files) json_processed_files = json.dumps(processed_files) - print(f"::set-output name=processed_files::{json_processed_files}") + print(json_processed_files) diff --git a/.github/workflows/run-tests-on-modified-meta.yml b/.github/workflows/run-tests-on-modified-meta.yml index 8c7416359..86932d896 100644 --- a/.github/workflows/run-tests-on-modified-meta.yml +++ b/.github/workflows/run-tests-on-modified-meta.yml @@ -1,4 +1,3 @@ -# This workflow will run configured tests for any updated MLC script name: Test script on modified meta on: @@ -11,40 +10,52 @@ jobs: get_modified_files: runs-on: ubuntu-latest outputs: - processed_files: ${{ steps.modified-files.outputs.processed_files }} + processed_files: ${{ steps.filter-modified-files.outputs.processed_files }} steps: - - name: 'Checkout' - uses: actions/checkout@v4 - with: - fetch-depth: 2 + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install pyyaml - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' + - name: Fetch base branch + run: | + git fetch origin +refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - name: Install dependencies - run: | - pip install pyyaml + - name: Get list of changed files + id: modified-files + run: | + git diff --name-only origin/${{ github.event.pull_request.base.ref }}...HEAD > changed_files.txt + files=$(paste -sd, changed_files.txt) + echo "files=$files" >> $GITHUB_OUTPUT - - name: Get changed files - id: modified-files - run: | - git remote add upstream ${{ github.event.pull_request.base.repo.clone_url }} - git fetch upstream - changed_files=$(git diff upstream/${{ github.event.pull_request.base.ref }} --name-only) - echo "$changed_files" | python3 .github/scripts/list_modified_files.py - - - name: Debug Show processed_files - run: | - echo "Processed files: '${{ steps.filter-modified-files.outputs.processed_files }}'" + - name: Filter changed files + id: filter-modified-files + env: + FILES: ${{ steps.modified-files.outputs.files }} + run: | + processed=$(echo "$FILES" | python3 .github/scripts/list_modified_files.py) + echo "processed_files<> $GITHUB_OUTPUT + echo "$processed" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Debug processed_files output + run: | + echo "Processed files output:" + echo "${{ steps.filter-modified-files.outputs.processed_files }}" process_modified_files: - runs-on: ubuntu-latest needs: get_modified_files - if: needs.determine_modified_files.outputs.processed_files != '[]' && needs.determine_modified_files.outputs.processed_files != '' + runs-on: ubuntu-latest + if: needs.get_modified_files.outputs.processed_files != '[]' strategy: fail-fast: false matrix: @@ -58,8 +69,7 @@ jobs: - name: Process meta.yaml file run: | - echo "Processing ${{ matrix.file_info.file }} with run number ${{ matrix.file_info.num_run }}" - + echo "Processing ${{ matrix.file_info.file }} (run #${{ matrix.file_info.num_run }})" pip install mlcflow mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }} - mlc test script ${{ matrix.file_info.uid}} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet + mlc test script ${{ matrix.file_info.uid }} --test_input_index=${{ matrix.file_info.num_run }} --docker_mlc_repo=${{ github.event.pull_request.head.repo.html_url }} --docker_mlc_repo_branch=${{ github.event.pull_request.head.ref }} --quiet diff --git a/script/get-lib-jemalloc/customize.py b/script/get-lib-jemalloc/customize.py new file mode 100644 index 000000000..a82d6a829 --- /dev/null +++ b/script/get-lib-jemalloc/customize.py @@ -0,0 +1,29 @@ +from mlc import utils +import os +import subprocess + + +def preprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + return {'return': 0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + os_info = i['os_info'] + + lib_path = os.path.join(os.getcwd(), "obj", "lib") + + env['+LD_LIBRARY_PATH'] = lib_path + env['MLC_JEMALLOC_LIB_PATH'] = lib_path + env['MLC_DEPENDENT_CACHED_PATH'] = os.path.join(lib_path, "libjemalloc.so") + + return {'return': 0} diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml new file mode 100644 index 000000000..929152152 --- /dev/null +++ b/script/get-lib-jemalloc/meta.yaml @@ -0,0 +1,45 @@ +alias: get-lib-jemalloc +automation_alias: script +automation_uid: 5b4e0237da074764 +category: Detection or installation of tools and artifacts +cache: true +deps: + - tags: get,git,repo + env: + MLC_GIT_CHECKOUT_PATH_ENV_NAME: MLC_JEMALLOC_SRC_PATH + force_env_keys: + - MLC_GIT_* + update_tags_from_env_with_prefix: + _branch.: + - MLC_GIT_CHECKOUT + _repo.: + - MLC_GIT_URL + _sha.: + - MLC_GIT_SHA + _submodules.: + - MLC_GIT_SUBMODULES + names: + - jemalloc-repo + extra_cache_tags: jemalloc,repo,jemalloc-repo +new_env_keys: + - MLC_JEMALLOC_LIB_PATH + - +LD_LIBRARY_PATH +new_state_keys: [] +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- get +- lib +- lib-jemalloc +- jemalloc +tests: + run_inputs: + - quiet: true +uid: 406439a446e04fb7 +variations: + version.official: + group: version + default: true + env: + MLC_GIT_URL: https://github.com/jemalloc/jemalloc.git diff --git a/script/get-lib-jemalloc/run.sh b/script/get-lib-jemalloc/run.sh new file mode 100644 index 000000000..2d156cf42 --- /dev/null +++ b/script/get-lib-jemalloc/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +#Add your run commands here... +# run "$MLC_RUN_CMD" +cd ${MLC_JEMALLOC_SRC_PATH} +autoconf +cd - +mkdir -p obj +cd obj +${MLC_JEMALLOC_SRC_PATH}/configure --enable-autogen +make