Adding our own CI (#216)

Setting up our CI to run on `master`. It has been rewritten to use make.
This PR also fixes and sets up `cargo clippy` to run as one of the
checks.

---------

Co-authored-by: Luis Eduardo de Souza Amorim <eduardo@rat.moma>
Co-authored-by: Yi Lin <qinsoon@gmail.com>

Author: 3 people

.github/scripts/Make.user

@@ -0,0 +1,2 @@
+FORCE_ASSERTIONS=1
+LLVM_ASSERTIONS=1

.github/scripts/ci-build.sh

@@ -0,0 +1,48 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+# Need a build_type argument
+if [ $# -eq 0 ]
+  then
+    echo "No arguments supplied"
+    exit 1
+fi
+# debug or release
+build_type=$1
+# plan to use
+plan=$2
+# moving vs non-moving
+is_moving=$3
+
+# helloworld.jl
+HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl
+
+# build MMTk
+build_args=""
+if [ "$build_type" == "release" ]; then
+    build_args=$build_args" --release"
+fi
+
+plan_feature=${plan,,}
+moving_feature=${is_moving,,}
+
+if [ "$is_moving" == "moving" ]; then
+    MOVING=1
+else
+    MOVING=0
+fi
+
+cd $MMTK_JULIA_DIR
+
+MMTK_MOVING=$MOVING make $build_type
+
+cd $JULIA_PATH
+
+# Clean first
+make cleanall
+# Build
+cp $BINDING_PATH/.github/scripts/Make.user $JULIA_PATH/
+MMTK_PLAN=$plan MMTK_BUILD=$build_type make
+# Run hello world
+$JULIA_PATH/julia $HELLO_WORLD_JL

.github/scripts/ci-checkout.sh

@@ -0,0 +1,12 @@
+set -ex
+
+. $(dirname "$0")/common.sh
+
+# We may later allow setting up a specific version of Julia using comments
+# in the PR, but for now we just use the latest master from JuliaLang
+JULIA_URL=https://github.com/$1.git
+JULIA_VERSION=$2
+
+rm -rf $JULIA_PATH
+git clone $JULIA_URL $JULIA_PATH
+git -C $JULIA_PATH checkout $JULIA_VERSION

.github/scripts/ci-ffi.sh

@@ -0,0 +1,10 @@
+set -ex
+
+. $(dirname "$0")/common.sh
+
+pushd $MMTK_JULIA_DIR
+
+make regen-bindgen-ffi
+
+popd
+

.github/scripts/ci-setup.sh

@@ -0,0 +1,9 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+rustup toolchain install $RUSTUP_TOOLCHAIN
+rustup target add i686-unknown-linux-gnu --toolchain $RUSTUP_TOOLCHAIN
+rustup component add clippy --toolchain $RUSTUP_TOOLCHAIN
+rustup component add rustfmt --toolchain $RUSTUP_TOOLCHAIN
+rustup override set $RUSTUP_TOOLCHAIN

.github/scripts/ci-style.sh

@@ -0,0 +1,13 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+export RUSTFLAGS="-D warnings"
+
+pushd $BINDING_PATH/mmtk
+
+cargo clippy
+cargo clippy --release
+
+cargo fmt -- --check
+popd

.github/scripts/ci-test-LinearAlgebra.sh

@@ -0,0 +1,16 @@
+# Running LinearAlgebra as a separate item
+# Given it takes on average more than 2.5h to run 
+
+set -e
+
+. $(dirname "$0")/common.sh
+
+export MMTK_MAX_HSIZE_G=8
+total_mem=$(free -m | awk '/^Mem:/ {print $2}')
+mem_threshold=512 # use 0.5Gb as a threshold for the max rss based on the total free memory
+total_mem_restricted=$((total_mem- mem_threshold))
+num_workers=2
+export JULIA_TEST_MAXRSS_MB=$((total_mem_restricted/ num_workers))
+
+echo "-> Run single threaded"
+ci_run_jl_test "LinearAlgebra" 2

.github/scripts/ci-test-gc-core.sh

@@ -0,0 +1,29 @@
+set -e
+
+# run tests that are more specific to the GC
+# this list was built intuitively based on tests that 
+# call the gc manually, have finalizers, have multiple threads, etc.
+
+. $(dirname "$0")/common.sh
+
+declare -a test_names=(
+    "gc"                            # Julia's GC specific testset
+    "threads"                       # Tests for multithreading (with some GC interaction)
+    "cmdlineargs"                   # Has tests that spawn other processes
+    "compiler"                      # Some tests for allocation, compiled code checks
+    "misc"                          # set of miscelaneous tests, include finalizers, eg
+    "core"                          # should have the core of the Julia code
+    "dict"                          # tests for weak references
+)
+
+for i in "${test_names[@]}"
+do
+    # echo "Token: '$i'"
+    test=`sed 's/\"\(.*\)\"/\1/' <<< $i`
+    if [[ ! -z "$test" ]]; then
+        echo $test
+
+        echo "-> Run"
+        ci_run_jl_test $test
+    fi
+done

.github/scripts/ci-test-other.sh

@@ -0,0 +1,38 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+# Get all the tests
+CHOOSE_TESTS_JL_PATH=$JULIA_PATH/test/choosetests.jl
+CHOOSE_TESTS_JL_CONTENT=`cat $CHOOSE_TESTS_JL_PATH`
+
+REGEX_PATTERN='.*const TESTNAMES = \[([^\[]*)^\].*'
+
+if [[ $CHOOSE_TESTS_JL_CONTENT =~ $REGEX_PATTERN ]]; then
+    RAW_TEST_NAMES=${BASH_REMATCH[1]}
+
+    readarray -td, test_names <<< "$RAW_TEST_NAMES"
+    declare test_names
+
+    for i in "${test_names[@]}"
+    do
+        # echo "Token: '$i'"
+        test=`sed 's/\"\(.*\)\"/\1/' <<< $i`
+        if [[ ! -z "$test" ]]; then
+            echo $test
+
+            # Should we skip some tests?
+            # Ignore stdlib tests for now -- we run stdlib tests separately
+            if [[ $test =~ "stdlib" ]]; then
+                echo "-> Skip stdlib"
+                continue
+            fi
+
+            echo "-> Run"
+            ci_run_jl_test $test
+        fi
+    done
+else
+    echo "Cannot find TESTNAMES in $CHOOSE_TESTS_JL_PATH"
+    exit 1
+fi

.github/scripts/ci-test-patching.sh

@@ -0,0 +1,71 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+# The list of tests/checks that we need to skip. They are either not suitable for Julia-MMTk, or
+# not supported at this moment.
+# Each line is a pattern of the test to match (we add skip=true to the end of those lines), and the test file path
+# * Pattern ends with $ so we won't append 'skip=true' multiple times
+declare -a tests_to_skip=(
+    # Ignore the entire libgit2.jl -- there are too many possible network related issues to run this test
+    # '@test.*$' "$JULIA_PATH/usr/share/julia/stdlib/v1.8/LibGit2/test/libgit2.jl"
+
+    # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+    '@test string(cpu_threads) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2,1 -e $code`, String) == "3"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "3"' "$JULIA_PATH/test/cmdlineargs.jl"
+
+    # These tests use the heapsize hint which is not used by mmtk
+    '@test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test abs(Float64(maxmem)' "$JULIA_PATH/test/cmdlineargs.jl"
+
+    # For some reason this fails even with the stock build
+    '@test n_precompiles <= expected_precompiles' "$JULIA_PATH/stdlib/REPL/test/precompilation.jl"
+    '@test length(targets) > 1' "$JULIA_PATH/test/precompile.jl"
+
+    # rr might not be available in the github runner
+    '@test success(pipeline(setenv(`$(Base.julia_cmd()) --bug-report=rr-local' "$JULIA_PATH/test/cmdlineargs.jl"
+
+    # These tests seem to fail because we set the number of stock GC threads to 0
+    'jl_setaffinity(1, mask, cpumasksize) == 0' "$JULIA_PATH/test/threads.jl"
+    'jl_getaffinity(1, mask, cpumasksize) == 0' "$JULIA_PATH/test/threads.jl"
+
+    # Skipping these GC tests for now (until we make sure we follow the stats as expected by the stock GC)
+    '@test !live_bytes_has_grown_too_much' "$JULIA_PATH/test/gc.jl"
+    '@test any(page_utilization .> 0)' "$JULIA_PATH/test/gc.jl"
+
+    # Tests that check the reasons for a full sweep and are specific to stock Julia
+    '@test reasons\[:FULL_SWEEP_REASON_FORCED_FULL_SWEEP\] >= 1' "$JULIA_PATH/test/gc.jl"
+    '@test keys(reasons) == Set(Base.FULL_SWEEP_REASONS)' "$JULIA_PATH/test/gc.jl"
+
+    # Allocation profiler tests that fail when we inline fastpath allocation
+    '@test length(\[a for a in prof.allocs if a.type == MyType\]) >= 1' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
+    '@test length(prof.allocs) >= 1' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
+    '@test length(filter(a->a.type <: type, profile.allocs)) >= NUM_TASKS' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
+    '@test length(profile.allocs) >= 2\*NUM_TASKS' "$JULIA_PATH/stdlib/Profile/test/allocs.jl"
+    
+    # Test that expects information from heap snapshot which is currently not available in MMTk
+    '@test contains(sshot, "redact_this")' "$JULIA_PATH/stdlib/Profile/test/runtests.jl"
+
+    # This test checks GC logging
+    '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
+
+    # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+    '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+    # This seems to be a regression from upstream when we merge with upstream 43bf2c8.
+    # The required string int.jl does not appear in the output even if I test with the stock Julia code.
+    # I do not know what is wrong, but at this point, I dont want to spend time on it.
+    '@test occursin("int.jl", code)' "$JULIA_PATH/test/cmdlineargs.jl"
+)
+
+for (( i=0; i < ${#tests_to_skip[@]}; i+=2 )); do
+    pattern=${tests_to_skip[i]}
+    file=${tests_to_skip[i+1]}
+    sed -i '/'"$pattern"'/ s/@test/@test_skip/' $file
+done