Skip to content

Commit c7097c5

Browse files
DilumAluthgestaticfloat
DilumAluthge
and
staticfloat
authored
CI (Buildkite): Add the package_linux64 and doctest builders (#41541)
Co-authored-by: Elliot Saba <staticfloat@gmail.com>
1 parent 0cbb685 commit c7097c5

File tree

7 files changed

+273
-15
lines changed

7 files changed

+273
-15
lines changed

‎.buildkite/misc/doctest.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
2+
# since we need nestable sandboxing. The rootfs images being used here are built from
3+
# the `.buildkite/rootfs_images/llvm-passes.jl` file.
4+
agents:
5+
queue: "julia"
6+
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
7+
sandbox.jl: "true"
8+
os: "linux"
9+
10+
steps:
11+
- label: "doctest"
12+
key: doctest
13+
plugins:
14+
- JuliaCI/julia#v1:
15+
version: 1.6
16+
- staticfloat/sandbox#v1:
17+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
18+
rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
19+
uid: 1000
20+
gid: 1000
21+
workspaces:
22+
# Include `/cache/repos` so that our `git` version introspection works.
23+
- "/cache/repos:/cache/repos"
24+
commands: |
25+
echo "--- Build Julia from source"
26+
make -j 6
27+
28+
echo "--- Print Julia version info"
29+
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
30+
31+
echo "--- Build Julia docs"
32+
make docs
33+
34+
echo "--- Run Julia doctests"
35+
JULIA_NUM_THREADS=1 make -C doc doctest=true
36+
timeout_in_minutes: 120
37+
notify:
38+
- github_commit_status:
39+
context: "doctest"

‎.buildkite/embedding.yml renamed to ‎.buildkite/misc/embedding.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ steps:
1414
- JuliaCI/julia#v1:
1515
version: 1.6
1616
- staticfloat/sandbox#v1:
17-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
18-
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
17+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
18+
rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
1919
uid: 1000
2020
gid: 1000
2121
workspaces:

‎.buildkite/llvm_passes.yml renamed to ‎.buildkite/misc/llvmpasses.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ steps:
1414
- JuliaCI/julia#v1:
1515
version: 1.6
1616
- staticfloat/sandbox#v1:
17-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
18-
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
17+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
18+
rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
1919
workspaces:
2020
# Include `/cache/repos` so that our `git` version introspection works.
2121
- "/cache/repos:/cache/repos"
@@ -36,8 +36,8 @@ steps:
3636
- JuliaCI/julia#v1:
3737
version: 1.6
3838
- staticfloat/sandbox#v1:
39-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
40-
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
39+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
40+
rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
4141
uid: 1000
4242
gid: 1000
4343
workspaces:

‎.buildkite/whitespace.yml renamed to ‎.buildkite/misc/whitespace.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ steps:
1414
- JuliaCI/julia#v1:
1515
version: 1.6
1616
- staticfloat/sandbox#v1:
17-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
18-
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
17+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
18+
rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
1919
workspaces:
2020
- "/cache/repos:/cache/repos"
2121
commands: |

‎.buildkite/pipeline.yml

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,25 @@
44
#
55
# Yes, this is creating another layer of indirection; the flow now looks like:
66
#
7-
# [webui] -> pipeline.yml -> llvm_passes.yml
7+
# [webui] -> pipeline.yml -> misc/whitespace.yml
88
#
9-
# when we could theoretically just have the `webui` launch `llvm_passes.yml`,
9+
# when we could theoretically just have the `webui` launch `misc/whitespace.yml`,
1010
# however this raises the bar for contributors to add new (unsigned) steps to
1111
# our CI configuration, so I'd rather live with an extra layer of indirection
1212
# and only need to touch the webui configuration when we need to alter
1313
# something about the privileged steps.
1414
steps:
1515
- label: ":buildkite: Launch unsigned pipelines"
1616
commands: |
17-
# We launch whitespace first, because we want that pipeline to finish as quickly as possible.
18-
# The remaining unsigned pipelines are launched in alphabetical order.
19-
buildkite-agent pipeline upload .buildkite/whitespace.yml
20-
buildkite-agent pipeline upload .buildkite/embedding.yml
21-
buildkite-agent pipeline upload .buildkite/llvm_passes.yml
17+
# First, we launch whitespace, because we want that pipeline to finish as quickly as possible.
18+
buildkite-agent pipeline upload .buildkite/misc/whitespace.yml
19+
20+
# Next, we launch the miscellaneous pipelines in alphabetical order.
21+
buildkite-agent pipeline upload .buildkite/misc/doctest.yml
22+
buildkite-agent pipeline upload .buildkite/misc/embedding.yml
23+
buildkite-agent pipeline upload .buildkite/misc/llvmpasses.yml
24+
25+
# Finally, we launch the platform pipelines in alphabetical order.
26+
buildkite-agent pipeline upload .buildkite/platforms/linux64.yml
2227
agents:
2328
queue: julia

‎.buildkite/platforms/linux64.yml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
2+
# since we need nestable sandboxing. The rootfs images being used here are built from
3+
# the `.buildkite/rootfs_images/llvm-passes.jl` file.
4+
agents:
5+
queue: "julia"
6+
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
7+
sandbox.jl: "true"
8+
os: "linux"
9+
10+
steps:
11+
- label: "package_linux64"
12+
key: package_linux64
13+
plugins:
14+
- JuliaCI/julia#v1:
15+
version: 1.6
16+
- staticfloat/sandbox#v1:
17+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
18+
rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
19+
uid: 1000
20+
gid: 1000
21+
workspaces:
22+
# Include `/cache/repos` so that our `git` version introspection works.
23+
- "/cache/repos:/cache/repos"
24+
commands: |
25+
echo "--- Build Julia from source"
26+
make -j 6
27+
make release
28+
make install
29+
30+
echo "--- Print Julia version info"
31+
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
32+
33+
echo "--- Compress build artifacts"
34+
mv julia-* julia-artifact
35+
rm -rf julia-linux64.tar.gz
36+
tar czf julia-linux64.tar.gz julia-artifact/
37+
38+
echo "--- Upload build artifacts"
39+
buildkite-agent artifact upload julia-linux64.tar.gz
40+
timeout_in_minutes: 60
41+
notify:
42+
- github_commit_status:
43+
context: "package_linux64"
44+
45+
# - label: "tester_linux64"
46+
# key: tester_linux64
47+
# depends_on: package_linux64
48+
# plugins:
49+
# - JuliaCI/julia#v1:
50+
# version: 1.6
51+
# - staticfloat/sandbox#v1:
52+
# # TODO: use a separate `tester_linux` image, instead of using the `package_linux` image.
53+
# rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
54+
# rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
55+
# uid: 1000
56+
# gid: 1000
57+
# workspaces:
58+
# # Include `/cache/repos` so that our `git` version introspection works.
59+
# - "/cache/repos:/cache/repos"
60+
# env:
61+
# JULIA_SHELL: "/bin/bash"
62+
# commands: |
63+
# echo "--- Download build artifacts"
64+
# rm -rf julia-linux64.tar.gz
65+
# buildkite-agent artifact download julia-linux64.tar.gz .
66+
#
67+
# echo "--- Extract build artifacts"
68+
# rm -rf julia-artifact/
69+
# tar xzf julia-linux64.tar.gz julia-artifact/
70+
#
71+
# echo "--- Print Julia version info"
72+
# julia-artifact/bin/julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
73+
#
74+
# echo "--- Run the Julia test suite"
75+
# unset JULIA_DEPOT_PATH
76+
# julia-artifact/bin/julia .buildkite/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)'
77+
# timeout_in_minutes: 120
78+
# notify:
79+
# - github_commit_status:
80+
# context: "tester_linux64"

‎.buildkite/rr_capture.jl

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
using Dates
2+
using Pkg
3+
using Tar
4+
5+
if Base.VERSION < v"1.6"
6+
throw(ErrorException("The `rr_capture.jl` script requires Julia 1.6 or greater"))
7+
end
8+
9+
if length(ARGS) < 1
10+
throw(ErrorException("Usage: rr_capture.jl [command...]"))
11+
end
12+
13+
const TIMEOUT = 2 * 60 * 60 # timeout in seconds
14+
15+
# We only use `rr` on the `tester_linux64` builder
16+
const use_rr_if_builder_is = "tester_linux64"
17+
18+
const run_id = get(ENV, "BUILDKITE_JOB_ID", "unknown")
19+
const shortcommit = get(ENV, "BUILDKITE_COMMIT", "unknown")
20+
const builder = get(ENV, "BUILDKITE_STEP_KEY", use_rr_if_builder_is)
21+
const use_rr = builder == use_rr_if_builder_is
22+
23+
@info "" run_id shortcommit builder use_rr
24+
@info "" ARGS
25+
26+
# if !use_rr # TODO: uncomment this line
27+
if true # TODO: delete this line
28+
@info "We will not run the tests under rr"
29+
p = run(`$ARGS`)
30+
exit(p.exitcode)
31+
end
32+
33+
@info "We will run the tests under rr"
34+
35+
const num_cores = min(Sys.CPU_THREADS, 8, parse(Int, get(ENV, "JULIA_TEST_NUM_CORES", "8")) + 1)
36+
@info "" num_cores
37+
38+
proc = nothing
39+
40+
new_env = copy(ENV)
41+
mktempdir() do dir
42+
Pkg.activate(dir)
43+
Pkg.add("rr_jll")
44+
Pkg.add("Zstd_jll")
45+
46+
rr_jll = Base.require(Base.PkgId(Base.UUID((0xe86bdf43_55f7_5ea2_9fd0_e7daa2c0f2b4)), "rr_jll"))
47+
zstd_jll = Base.require(Base.PkgId(Base.UUID((0x3161d3a3_bdf6_5164_811a_617609db77b4)), "Zstd_jll"))
48+
rr(func) = Base.invokelatest(rr_jll.rr, func; adjust_LIBPATH=false)
49+
rr() do rr_path
50+
capture_script_path = joinpath(dir, "capture_output.sh")
51+
loader = Sys.WORD_SIZE == 64 ? "/lib64/ld-linux-x86-64.so.2" : "/lib/ld-linux.so.2"
52+
open(capture_script_path, "w") do io
53+
write(io, """
54+
#!/bin/bash
55+
56+
$(rr_path) record --nested=detach "\$@" > >(tee -a $(dir)/stdout.log) 2> >(tee -a $(dir)/stderr.log >&2)
57+
""")
58+
end
59+
chmod(capture_script_path, 0o755)
60+
61+
new_env = copy(ENV)
62+
new_env["_RR_TRACE_DIR"] = joinpath(dir, "rr_traces")
63+
new_env["RR_LOG"]="all:debug"
64+
new_env["RR_LOG_BUFFER"]="100000"
65+
new_env["JULIA_RR"] = capture_script_path
66+
t_start = time()
67+
global proc = run(setenv(`$(rr_path) record --num-cores=$(num_cores) $ARGS`, new_env), (stdin, stdout, stderr); wait=false)
68+
69+
# Start asynchronous timer that will kill `rr`
70+
@async begin
71+
sleep(TIMEOUT)
72+
73+
# If we've exceeded the timeout and `rr` is still running, kill it.
74+
if isopen(proc)
75+
println(stderr, "\n\nProcess timed out. Signalling `rr` for force-cleanup!")
76+
kill(proc, Base.SIGTERM)
77+
78+
# Give `rr` a chance to cleanup
79+
sleep(60)
80+
81+
if isopen(proc)
82+
println(stderr, "\n\n`rr` failed to cleanup within one minute, killing and exiting immediately!")
83+
kill(proc, Base.SIGKILL)
84+
exit(1)
85+
end
86+
end
87+
end
88+
89+
# Wait for `rr` to finish, either through naturally finishing its run, or `SIGTERM`.
90+
# If we have to `SIGKILL`
91+
wait(proc)
92+
93+
# On a non-zero exit code, upload the `rr` trace
94+
if !success(proc)
95+
println(stderr, "`rr` returned $(proc.exitcode), packing and uploading traces...")
96+
97+
if !isdir(joinpath(dir, "rr_traces"))
98+
println(stderr, "No `rr_traces` directory! Did `rr` itself fail?")
99+
exit(1)
100+
end
101+
102+
# Clean up non-traces
103+
rm(joinpath(dir, "rr_traces", "latest-trace"))
104+
rm(joinpath(dir, "rr_traces", "cpu_lock"))
105+
106+
# Create a directory for the pack files to go
107+
pack_dir = joinpath(dir, "pack")
108+
mkdir(pack_dir)
109+
110+
# Pack all traces
111+
trace_dirs = [joinpath(dir, "rr_traces", f) for f in readdir(joinpath(dir, "rr_traces"))]
112+
filter!(isdir, trace_dirs)
113+
run(ignorestatus(`$(rr_path) pack --pack-dir=$pack_dir $(trace_dirs)`))
114+
115+
# Tar it up
116+
mkpath("dumps")
117+
datestr = Dates.format(now(), dateformat"yyyy-mm-dd_HH_MM_SS")
118+
dst_path = "dumps/rr-run_$(run_id)-gitsha_$(shortcommit)-$(datestr).tar.zst"
119+
zstd_jll.zstdmt() do zstdp
120+
tarproc = open(`$zstdp -o $dst_path`, "w")
121+
Tar.create(dir, tarproc)
122+
close(tarproc.in)
123+
end
124+
end
125+
end
126+
end
127+
128+
# Pass the exit code back up to Buildkite
129+
if proc.termsignal != 0
130+
ccall(:raise, Cvoid, (Cint,), proc.termsignal)
131+
exit(1) # Just in case the signal did not cause an exit
132+
else
133+
exit(proc.exitcode)
134+
end

0 commit comments

Comments
 (0)