Skip to content

Adding CI support #256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: mmtk-support-moving-upstream
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .github/scripts/ci-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ fi
build_type=$1
# plan to use
plan=$2
# moving vs non-moving
# max-moving vs non-moving
is_moving=$3

# helloworld.jl
Expand All @@ -27,10 +27,14 @@ fi
plan_feature=${plan,,}
moving_feature=${is_moving,,}

if [ "$is_moving" == "moving" ]; then
if [ "$moving_feature" == "max_moving" ]; then
MOVING=1
ALWAYS_MOVING=1
MAX_MOVING=1
else
MOVING=0
ALWAYS_MOVING=0
MAX_MOVING=0
fi

cd $JULIA_PATH
Expand All @@ -39,6 +43,6 @@ make cleanall
# This will build the binding in MMTK_JULIA_DIR (defined in common.sh), and link it
# when building Julia, instead of using the set version defined in Julia itself
cp $BINDING_PATH/.github/scripts/Make.user $JULIA_PATH/
MMTK_MOVING=$MOVING MMTK_PLAN=$plan MMTK_BUILD=$build_type make
MMTK_MOVING=$MOVING MMTK_ALWAYS_MOVING=$ALWAYS_MOVING MMTK_MAX_MOVING=$MAX_MOVING MMTK_PLAN=$plan MMTK_BUILD=$build_type make
# Run hello world
$JULIA_PATH/julia $HELLO_WORLD_JL
6 changes: 5 additions & 1 deletion .github/scripts/ci-test-LinearAlgebra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,9 @@ total_mem_restricted=$((total_mem- mem_threshold))
num_workers=2
export JULIA_TEST_MAXRSS_MB=$((total_mem_restricted/ num_workers))

# max-moving vs non-moving
is_moving=$2
moving_feature=${is_moving,,}

echo "-> Run single threaded"
ci_run_jl_test "LinearAlgebra" 2
ci_run_jl_test "LinearAlgebra" 2 $moving_feature
6 changes: 5 additions & 1 deletion .github/scripts/ci-test-other.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ CHOOSE_TESTS_JL_CONTENT=`cat $CHOOSE_TESTS_JL_PATH`

REGEX_PATTERN='.*const TESTNAMES = \[([^\[]*)^\].*'

# max-moving vs non-moving
is_moving=$2
moving_feature=${is_moving,,}

if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
RAW_TEST_NAMES=${BASH_REMATCH[1]}

Expand Down Expand Up @@ -40,7 +44,7 @@ if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
fi

echo "-> Run"
ci_run_jl_test $test
ci_run_jl_test $test 1 $moving_feature
fi
done
else
Expand Down
10 changes: 7 additions & 3 deletions .github/scripts/ci-test-stdlib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ if [ $(find $stdlib_path -mindepth 1 -maxdepth 1 | wc -l) -ne 1 ]; then
exit 1
fi

# max-moving vs non-moving
is_moving=$2
moving_feature=${is_moving,,}

for dir in $(find $stdlib_version_path -depth -mindepth 1 -type d -o -type l)
do
# if there is a runtests.jl, we run it.
Expand All @@ -67,16 +71,16 @@ do

if [[ "${tests_with_multi_workers[@]}" =~ "$test" ]]; then
echo "-> Run multi threaded"
ci_run_jl_test $test 2
ci_run_jl_test $test 2 $moving_feature
continue
fi

if [[ "${tests_with_single_worker[@]}" =~ "$test" ]]; then
echo "-> Run single threaded"
ci_run_jl_test $test 1
ci_run_jl_test $test 1 $moving_feature
continue
fi

ci_run_jl_test $test
ci_run_jl_test $test 1 $moving_feature
fi
done
14 changes: 13 additions & 1 deletion .github/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,18 @@ export JULIA_TEST_MAXRSS_MB=$total_mem
ci_run_jl_test() {
test=$1
threads=$2
moving_feature=$3


if [ "$moving_feature" == "max_moving" ]; then
MOVING=1
ALWAYS_MOVING=1
MAX_MOVING=1
else
MOVING=0
ALWAYS_MOVING=0
MAX_MOVING=0
fi

# if no argument is given, use 2 as default
if [ -z "$threads" ]; then
Expand All @@ -30,5 +42,5 @@ ci_run_jl_test() {
# $JULIA_PATH/julia $JULIA_TEST_ARGS $JULIA_PATH/test/runtests.jl --exit-on-error $test

# Run with their build script
make test-$test
MMTK_MOVING=$MOVING MMTK_ALWAYS_MOVING=$ALWAYS_MOVING MMTK_MAX_MOVING=$MAX_MOVING make test-$test
}
95 changes: 48 additions & 47 deletions .github/workflows/binding-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,51 +39,52 @@ jobs:
run: |
./.github/scripts/ci-style.sh

build-debug-regen-ffi:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- name: Setup environments
run: |
./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
./.github/scripts/ci-setup.sh
# removing these as they cause a conflict within bindgen
sudo rm -rf /usr/lib/llvm-14
sudo rm -rf /usr/lib/llvm-13
- name: Check FFI bindings
run: |
./.github/scripts/ci-ffi.sh
- name: Build Julia (Debug)
run: |
./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
- name: Print Julia version info
run: |
JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
- name: Style check
run: |
./.github/scripts/ci-style.sh
# Commenting these out since they shouldn't be needed in the mmtk-support-moving-upstream branch
# build-debug-regen-ffi:
# runs-on: ubuntu-22.04
# steps:
# - uses: actions/checkout@v2
# - name: Setup environments
# run: |
# ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
# ./.github/scripts/ci-setup.sh
# # removing these as they cause a conflict within bindgen
# sudo rm -rf /usr/lib/llvm-14
# sudo rm -rf /usr/lib/llvm-13
# - name: Check FFI bindings
# run: |
# ./.github/scripts/ci-ffi.sh
# - name: Build Julia (Debug)
# run: |
# ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }} ${{ inputs.moving }}
# - name: Print Julia version info
# run: |
# JULIA_PATH=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia/vm/julia
# $JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
# - name: Style check
# run: |
# ./.github/scripts/ci-style.sh

check-ffi:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- name: Setup environments
run: |
./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
./.github/scripts/ci-setup.sh
# removing these as they cause a conflict within bindgen
sudo rm -rf /usr/lib/llvm-14
sudo rm -rf /usr/lib/llvm-13
- name: Regenerate FFI bindings
run: |
./.github/scripts/ci-ffi.sh
- run: |
MMTK_JULIA_DIR=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia
if ! git diff --exit-code $MMTK_JULIA_DIR/mmtk/src/julia_types.rs; then
echo "Rust FFI bindings in \`julia_types.rs\` are outdated. Run \`make regen-bindgen-ffi\` from the mmtk-julia directory and make sure to include the updated file in the pull request."
exit 1
fi
# check-ffi:
# runs-on: ubuntu-22.04
# steps:
# - uses: actions/checkout@v2
# - name: Setup environments
# run: |
# ./.github/scripts/ci-checkout.sh ${{ inputs.julia_repo }} ${{ inputs.julia_ref }}
# ./.github/scripts/ci-setup.sh
# # removing these as they cause a conflict within bindgen
# sudo rm -rf /usr/lib/llvm-14
# sudo rm -rf /usr/lib/llvm-13
# - name: Regenerate FFI bindings
# run: |
# ./.github/scripts/ci-ffi.sh
# - run: |
# MMTK_JULIA_DIR=$(realpath $(dirname "$0"))/../mmtk-julia/mmtk-julia
# if ! git diff --exit-code $MMTK_JULIA_DIR/mmtk/src/julia_types.rs; then
# echo "Rust FFI bindings in \`julia_types.rs\` are outdated. Run \`make regen-bindgen-ffi\` from the mmtk-julia directory and make sure to include the updated file in the pull request."
# exit 1
# fi

build-test-other:
runs-on: ubuntu-22.04
Expand All @@ -109,7 +110,7 @@ jobs:
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
- name: Test Julia
run: |
./.github/scripts/ci-test-other.sh ${{ inputs.gc_plan }}
./.github/scripts/ci-test-other.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}

build-test-stdlib:
runs-on: ubuntu-22.04
Expand All @@ -135,7 +136,7 @@ jobs:
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
- name: Test Julia
run: |
./.github/scripts/ci-test-stdlib.sh ${{ inputs.gc_plan }}
./.github/scripts/ci-test-stdlib.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}

build-test-LinearAlgebra:
runs-on: ubuntu-22.04
Expand All @@ -158,4 +159,4 @@ jobs:
$JULIA_PATH/julia --project=. -e "using InteractiveUtils; versioninfo()"
- name: Test Julia
run: |
./.github/scripts/ci-test-LinearAlgebra.sh ${{ inputs.gc_plan }}
./.github/scripts/ci-test-LinearAlgebra.sh ${{ inputs.gc_plan }} ${{ inputs.moving }}
7 changes: 4 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
- v1.8.2\+RAI
- v1.9.2\+RAI
- dev
- mmtk-support-moving-upstream

concurrency:
# Cancels pending runs when a PR gets updated.
Expand All @@ -17,8 +18,8 @@ jobs:
julia-version:
runs-on: ubuntu-latest
env:
JULIA_REPO_DEFAULT: JuliaLang/julia
JULIA_REF_DEFAULT: master
JULIA_REPO_DEFAULT: mmtk/julia
JULIA_REF_DEFAULT: mmtk-support-moving-upstream
outputs:
julia_repo: ${{ steps.print.outputs.julia_repo }}
julia_ref: ${{ steps.print.outputs.julia_ref }}
Expand All @@ -40,7 +41,7 @@ jobs:
fail-fast: false
matrix:
gc_plan: [Immix]
moving: [Non_Moving, Moving]
moving: [Non_Moving, Max_Moving]
uses: ./.github/workflows/binding-tests.yml
with:
gc_plan: ${{ matrix.gc_plan }}
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ release:
@cd $(MMTK_JULIA_DIR)mmtk && $(PROJECT_DIRS) cargo build --features $(CARGO_FEATURES) --release

debug:
@echo "Building the Rust project in $(MMTK_JULIA_DIR) using a debug build";
@echo "Building the Rust project in $(MMTK_JULIA_DIR)mmtk using a debug build with MMTK_VARS: $(MMTK_VARS)";
@cd $(MMTK_JULIA_DIR)mmtk && $(PROJECT_DIRS) cargo build --features $(CARGO_FEATURES)

# Build the Julia project (which will build the binding as part of their deps build)
Expand Down
Loading