Skip to content

Commit 9a1ceb9

Browse files
asbasb
authored
Move first RISC-V builder over to cross-compiler + execute tests under qemu-system setup (#279)
1 parent eb2686e commit 9a1ceb9

File tree

4 files changed

+285
-12
lines changed

4 files changed

+285
-12
lines changed

‎buildbot/osuosl/master/config/builders.py

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3246,16 +3246,20 @@
32463246
'CXX': 'clang++',
32473247
})},
32483248

3249-
## RISC-V RVA20 profile check-all 2-stage
3249+
## RISC-V RVA20 profile check-all 2-stage. The second stage is
3250+
# cross-compiled on the x86 host and then lit runs under a qemu-system image
3251+
# using the just-built artifacts.
32503252
{'name' : "clang-riscv-rva20-2stage",
32513253
'tags' : ["clang"],
32523254
'workernames' : ["rise-clang-riscv-rva20-2stage"],
32533255
'builddir':"clang-riscv-rva20-2stage",
32543256
'factory' : ClangBuilder.getClangCMakeBuildFactory(
3255-
clean=False,
3257+
clean=True,
32563258
useTwoStage=True,
32573259
runTestSuite=False,
32583260
testStage1=False,
3261+
checkout_compiler_rt=False,
3262+
checkout_zorg=True,
32593263
extra_cmake_args=[
32603264
"-DCMAKE_C_COMPILER=clang",
32613265
"-DCMAKE_CXX_COMPILER=clang++",
@@ -3264,9 +3268,26 @@
32643268
"-DCMAKE_C_COMPILER_LAUNCHER=ccache",
32653269
"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"],
32663270
extra_stage2_cmake_args=[
3267-
"-DLLVM_ENABLE_LLD=True",
3268-
"-DCMAKE_C_FLAGS='-march=rva20u64'",
3269-
"-DCMAKE_CXX_FLAGS='-march=rva20u64'"]
3271+
util.Interpolate("-DLLVM_NATIVE_TOOL_DIR=%(prop:builddir)s/stage1.install/bin"),
3272+
"-DLLVM_BUILD_TESTS=True",
3273+
util.Interpolate("-DLLVM_EXTERNAL_LIT=%(prop:builddir)s/llvm-zorg/buildbot/riscv-rise/lit-on-qemu")],
3274+
stage2_toolchain_options=[
3275+
"set(CMAKE_SYSTEM_NAME Linux)",
3276+
"set(CMAKE_SYSROOT %(prop:builddir)s/../rvsysroot)",
3277+
"set(CMAKE_C_COMPILER_TARGET riscv64-linux-gnu)",
3278+
"set(CMAKE_CXX_COMPILER_TARGET riscv64-linux-gnu)",
3279+
"set(CMAKE_C_FLAGS_INIT '-march=rva20u64')",
3280+
"set(CMAKE_CXX_FLAGS_INIT '-march=rva20u64')",
3281+
"set(CMAKE_LINKER_TYPE LLD)",
3282+
"set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)",
3283+
"set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)",
3284+
"set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)",
3285+
"set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)"],
3286+
env={
3287+
"BB_IMG_DIR": util.Interpolate("%(prop:builddir)s/.."),
3288+
"BB_QEMU_CPU": "rv64,zfa=false,zba=false,zbb=false,zbc=false,zbs=false",
3289+
"BB_QEMU_SMP": "32",
3290+
"BB_QEMU_MEM": "64G"}
32703291
)},
32713292

32723293
## RISC-V RVA23 profile check-all 2-stage

‎buildbot/riscv-rise/create-sid-riscv-appliance-image.sh

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/sh
2+
#===----------------------------------------------------------------------===//
3+
#
4+
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5+
# See https://llvm.org/LICENSE.txt for license information.
6+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7+
#
8+
#===----------------------------------------------------------------------===//
9+
10+
TGT=riscv-sid-for-qemu
11+
# Uses <https://github.com/muxup/medley/blob/main/rootless-debootstrap-wrapper>
12+
~/rootless-debootstrap-wrapper \
13+
--arch=riscv64 \
14+
--suite=sid \
15+
--cache-dir="$HOME/debcache" \
16+
--target-dir=$TGT \
17+
--include=linux-image-riscv64,zstd,dbus,adduser,python3,python3-psutil,git
18+
cat - <<EOF > $TGT/etc/resolv.conf
19+
nameserver 1.1.1.1
20+
EOF
21+
"$TGT/_enter" sh -e <<'EOF'
22+
ln -sf /dev/null /etc/udev/rules.d/80-net-setup-link.rules # disable persistent network names
23+
cat - <<INNER_EOF > /etc/systemd/network/10-eth0.network
24+
[Match]
25+
Name=eth0
26+
27+
[Network]
28+
DHCP=yes
29+
INNER_EOF
30+
systemctl enable systemd-networkd
31+
echo root:root | chpasswd
32+
adduser --gecos ",,," --disabled-password user
33+
echo user:user | chpasswd
34+
ln -sf /dev/null /etc/systemd/system/serial-getty@hvc0.service
35+
ln -sf /dev/null /etc/systemd/system/serial-getty@ttyS0.service
36+
37+
cat - <<'INNER_EOF' > /opt/on-boot-logic.sh
38+
#!/bin/sh
39+
error() {
40+
printf "!!!!!!!!!! Error: %s !!!!!!!!!!\n" "$*" >&2
41+
exit 1
42+
}
43+
44+
mkdir -p /mnt/hgcomm
45+
mount -t 9p -o trans=virtio,version=9p2000.L hgcomm /mnt/hgcomm || error "Failed to mount hgcomm"
46+
if [ -e /mnt/hgcomm/debug-mode-on ]; then
47+
echo "debug-mode-on file present: Not executing exec-on-boot and instead starting getty"
48+
systemctl unmask serial-getty@ttyS0.service
49+
systemctl start serial-getty@ttyS0.service
50+
systemctl mask serial-getty@ttyS0.service
51+
exit 0
52+
fi
53+
[ -f /mnt/hgcomm/exec-on-boot ] || error "exec-on-boot doesn't exist"
54+
[ -x /mnt/hgcomm/exec-on-boot ] || error "exec-on-boot isn't executable"
55+
/mnt/hgcomm/exec-on-boot
56+
echo "$?" > /mnt/hgcomm/exec-on-boot.exitcode
57+
poweroff
58+
INNER_EOF
59+
chmod +x /opt/on-boot-logic.sh
60+
61+
cat - <<'INNER_EOF' > /etc/systemd/system/appliance.service
62+
[Unit]
63+
Description=Execute on boot logic
64+
After=multi-user.target
65+
66+
[Service]
67+
Type=oneshot
68+
StandardOutput=tty
69+
TTYPath=/dev/ttyS0
70+
ExecStart=/opt/on-boot-logic.sh
71+
ExecStopPost=/bin/sh -c '[ "$EXIT_STATUS" != 0 ] && poweroff'
72+
73+
[Install]
74+
WantedBy=multi-user.target
75+
INNER_EOF
76+
systemctl enable appliance.service
77+
echo "Finished rootfs config"
78+
EOF
79+
80+
fakeroot -i $TGT/.fakeroot.env sh <<EOF
81+
ln -L $TGT/vmlinuz kernel
82+
ln -L $TGT/initrd.img initrd
83+
fallocate -l 30GiB rootfs.img
84+
mkfs.ext4 -d $TGT rootfs.img
85+
EOF

‎buildbot/riscv-rise/lit-on-qemu

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/usr/bin/env python3
2+
import os
3+
import pathlib
4+
import shutil
5+
import subprocess
6+
import sys
7+
8+
# Note:
9+
# * Builders always use the latest version of this script, checking out the
10+
# llvm-zorg repository. This means you can't rely on the script being updated
11+
# in lockstep with the buildmaster being re-deployed.
12+
# * This script expects to be run from within an LLVM build directory. e.g.
13+
# llvm-project/build/foo.
14+
# * The approach in this script could easily be applied to other
15+
# architectures. It may or may not make sense to generalise this script in
16+
# order to do so. This is intentionally left for evaluation when/if someone
17+
# applies seeks to apply the same approach on another target.
18+
19+
20+
def error(message):
21+
print(f"!!!!!!!!!! Error: {message} !!!!!!!!!!")
22+
sys.exit(1)
23+
24+
25+
# Validate environment variables
26+
for var in ["BB_IMG_DIR", "BB_QEMU_CPU", "BB_QEMU_SMP", "BB_QEMU_MEM"]:
27+
if not os.getenv(var):
28+
error(f"{var} not set")
29+
30+
# Create appropriate exec-on-boot script
31+
current_path = pathlib.Path.cwd()
32+
build_dir_name = current_path.name
33+
hgcomm_path = current_path / "hgcomm"
34+
35+
if hgcomm_path.exists():
36+
shutil.rmtree(hgcomm_path)
37+
hgcomm_path.mkdir()
38+
39+
target_uid = 1000
40+
target_gid = 1000
41+
base_mount_path = current_path.parent
42+
43+
args_string = " ".join(sys.argv[1:])
44+
exec_on_boot_content = f"""#!/bin/sh
45+
error() {{
46+
printf "!!!!!!!!!! Error: %s !!!!!!!!!!\\n" "$*" >&2
47+
exit 1
48+
}}
49+
[ -e "{base_mount_path}" ] && error "Can't mount path, already exists"
50+
mkdir -p "{base_mount_path}" || error "Can't make mount path"
51+
chown {target_uid}:{target_gid} "{base_mount_path}" || error "Chown failed"
52+
mount -t ext4 /dev/vdb "{base_mount_path}" || error "Mount failed"
53+
cd "{current_path}"
54+
su user -c "./bin/llvm-lit {args_string}"
55+
"""
56+
exec_on_boot_path = hgcomm_path / "exec-on-boot"
57+
exec_on_boot_path.write_text(exec_on_boot_content)
58+
exec_on_boot_path.chmod(0o755)
59+
60+
# Create ext4 filesystem containing the LLVM build directory and LLVM source
61+
# tree. Two layouts are supported:
62+
# 1) A typical layout used in local development, with build directories within
63+
# build/ in the monorepo checkout.
64+
# 2) The layout used by ClangBuilder.py in CI, with build directories as
65+
# siblings to the monorepo checkout (named 'llvm').
66+
print("@@@@@@@@@@ Creating ext4 filesystem with LLVM build directory @@@@@@@@")
67+
subprocess.run(["fallocate", "-l", "25GiB", "llvm-project.img"], check=True)
68+
69+
if (current_path.parent.parent / ".git").is_dir():
70+
print("Note: 'Local dev' layout detected (build/build_dir nested in LLVM checkout)")
71+
extra_tar_args = [
72+
f"--exclude=build/{p.name} "
73+
for p in current_path.parent.iterdir()
74+
if p.is_dir() and p.name != build_dir_name
75+
]
76+
extra_tar_args.append("--exclude=.git")
77+
extra_tar_args.append(f"--exclude=build/{build_dir_name}/llvm-project.img")
78+
paths_to_tar = "."
79+
change_to_dir = "../.."
80+
elif (current_path.parent / "llvm" / ".git").is_dir():
81+
print("Note: 'CI style' layout detected (llvm checkout and build_dir as siblings)")
82+
extra_tar_args = [
83+
"--exclude=llvm/.git",
84+
f"--exclude={build_dir_name}/llvm-project.img"
85+
]
86+
paths_to_tar = f"llvm {build_dir_name}"
87+
change_to_dir = ".."
88+
else:
89+
error("Unrecognized repo/build layout")
90+
91+
parent_dir = current_path.parent
92+
tar_command = (
93+
f"tar --create --file=- --owner={target_uid} --group={target_gid} "
94+
f"{' '.join(extra_tar_args)} "
95+
f"-C {change_to_dir} {paths_to_tar} | mkfs.ext4 -d - llvm-project.img"
96+
)
97+
subprocess.run(tar_command, shell=True, check=True)
98+
99+
# Launch qemu-system appliance
100+
print("@@@@@@@@@@ Pivoting execution to qemu-system @@@@@@@@")
101+
# fmt: off
102+
qemu_command = [
103+
"qemu-system-riscv64",
104+
"-machine", "virt",
105+
"-cpu", os.getenv("BB_QEMU_CPU"),
106+
"-smp", os.getenv("BB_QEMU_SMP"),
107+
"-m", os.getenv("BB_QEMU_MEM"),
108+
"-device", "virtio-blk-device,drive=hd",
109+
"-drive", f"file={os.getenv('BB_IMG_DIR')}/rootfs.img,if=none,id=hd,format=raw",
110+
"-virtfs", "local,path=hgcomm,mount_tag=hgcomm,security_model=none,id=hgcomm",
111+
"-device", "virtio-blk-device,drive=hdb",
112+
"-drive", "file=llvm-project.img,format=raw,if=none,id=hdb",
113+
"-device", "virtio-net-device,netdev=net",
114+
"-netdev", "user,id=net,hostfwd=tcp:127.0.0.1:10222-:22",
115+
"-bios", "/usr/share/qemu/opensbi-riscv64-generic-fw_dynamic.bin",
116+
"-kernel", f"{os.getenv('BB_IMG_DIR')}/kernel",
117+
"-initrd", f"{os.getenv('BB_IMG_DIR')}/initrd",
118+
"-object", "rng-random,filename=/dev/urandom,id=rng",
119+
"-device", "virtio-rng-device,rng=rng",
120+
"-nographic",
121+
"-append", "rw quiet root=/dev/vda console=ttyS0",
122+
]
123+
# fmt: on
124+
subprocess.run(qemu_command, check=True)
125+
print("@@@@@@@@@@ qemu-system execution finished @@@@@@@@")
126+
127+
exit_code_file = hgcomm_path / "exec-on-boot.exitcode"
128+
if exit_code_file.is_file():
129+
sys.exit(int(exit_code_file.read_text().strip()))
130+
else:
131+
sys.exit(111)

‎zorg/buildbot/builders/ClangBuilder.py

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from buildbot.plugins import util
55
from buildbot.steps.shell import ShellCommand, SetProperty
66
from buildbot.steps.shell import WarningCountingShellCommand
7+
from buildbot.steps.transfer import StringDownload
78

89
import zorg.buildbot.builders.Util as builders_util
910

@@ -162,13 +163,18 @@ def getClangCMakeBuildFactory(
162163
# CMake arguments to use for stage2 instead of extra_cmake_args.
163164
extra_stage2_cmake_args=None,
164165

166+
# If set, use a toolchains file for the stage 2 build and add
167+
# these options to it.
168+
stage2_toolchain_options=None,
169+
165170
# Extra repositories
166171
checkout_clang_tools_extra=True,
167172
checkout_compiler_rt=True,
168173
checkout_lld=True,
169174
checkout_libcxx=False,
170175
checkout_flang=False,
171176
checkout_test_suite=False,
177+
checkout_zorg=False,
172178

173179
enable_runtimes="auto"):
174180
return _getClangCMakeBuildFactory(
@@ -180,12 +186,14 @@ def getClangCMakeBuildFactory(
180186
submitURL=submitURL, testerName=testerName,
181187
env=env, extra_cmake_args=extra_cmake_args,
182188
extra_stage2_cmake_args=extra_stage2_cmake_args,
189+
stage2_toolchain_options=stage2_toolchain_options,
183190
checkout_clang_tools_extra=checkout_clang_tools_extra,
184191
checkout_lld=checkout_lld,
185192
checkout_compiler_rt=checkout_compiler_rt,
186193
checkout_libcxx=checkout_libcxx,
187194
checkout_flang=checkout_flang,
188195
checkout_test_suite=checkout_test_suite,
196+
checkout_zorg=checkout_zorg,
189197
enable_runtimes=enable_runtimes)
190198

191199
def _getClangCMakeBuildFactory(
@@ -219,13 +227,18 @@ def _getClangCMakeBuildFactory(
219227
# CMake arguments to use for stage2 instead of extra_cmake_args.
220228
extra_stage2_cmake_args=None,
221229

230+
# If set, use a toolchains file for the stage 2 build and add
231+
# these options to it.
232+
stage2_toolchain_options=None,
233+
222234
# Extra repositories
223235
checkout_clang_tools_extra=True,
224236
checkout_compiler_rt=True,
225237
checkout_lld=True,
226238
checkout_libcxx=False,
227239
checkout_test_suite=False,
228240
checkout_flang=False,
241+
checkout_zorg=False,
229242

230243
enable_runtimes="auto",
231244

@@ -294,6 +307,12 @@ def _getClangCMakeBuildFactory(
294307
src_dir='test/test-suite',
295308
alwaysUseLatest=True)
296309

310+
if checkout_zorg:
311+
f.addGetSourcecodeForProject(
312+
project='zorg',
313+
src_dir='llvm-zorg',
314+
alwaysUseLatest=True)
315+
297316
# Then get the LLVM source code revision this particular build is for.
298317
f.addGetSourcecodeSteps()
299318

@@ -427,10 +446,28 @@ def _getClangCMakeBuildFactory(
427446
# Absolute paths to just built compilers.
428447
# Note: Backslash path separators do not work well with cmake and ninja.
429448
# Forward slash path separator works on Windows as well.
430-
stage1_cc = InterpolateToPosixPath(
431-
f"-DCMAKE_C_COMPILER=%(prop:builddir)s/{stage1_install}/bin/{cc}")
432-
stage1_cxx = InterpolateToPosixPath(
433-
f"-DCMAKE_CXX_COMPILER=%(prop:builddir)s/{stage1_install}/bin/{cxx}")
449+
stage1_cc = f"%(prop:builddir)s/{stage1_install}/bin/{cc}"
450+
stage1_cxx = f"%(prop:builddir)s/{stage1_install}/bin/{cxx}"
451+
452+
# If stage2_toolchain_options is set then we'll use a toolchain file
453+
# to specify the compiler being used (the just-built stage1) and add
454+
# any stage2_toolchain_options to it. Otherwise, just set
455+
# -DCMAKE_{C,CXX}_COMPILER.
456+
if stage2_toolchain_options is None:
457+
compiler_args = [
458+
InterpolateToPosixPath(f"-DCMAKE_C_COMPILER={stage1_cc}"),
459+
InterpolateToPosixPath(f"-DCMAKE_CXX_COMPILER={stage1_cxx}"),
460+
]
461+
else:
462+
toolchain_file = f"%(prop:builddir)s/{stage2_build}/stage1-toolchain.cmake"
463+
toolchain_file_contents = "\n".join([
464+
f"set(CMAKE_C_COMPILER {stage1_cc})",
465+
f"set(CMAKE_CXX_COMPILER {stage1_cxx})",
466+
] + stage2_toolchain_options)
467+
f.addStep(StringDownload(util.Interpolate(toolchain_file_contents),
468+
workerdest=InterpolateToPosixPath(toolchain_file)))
469+
compiler_args = [InterpolateToPosixPath(f"-DCMAKE_TOOLCHAIN_FILE={toolchain_file}")]
470+
434471

435472
# If we have a separate stage2 cmake arg list, then ensure we re-apply
436473
# enable_projects and enable_runtimes if necessary.
@@ -446,13 +483,12 @@ def _getClangCMakeBuildFactory(
446483

447484
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage2_build)
448485
cmake_cmd2 = [cmake, "-G", "Ninja", rel_src_dir,
449-
stage1_cc,
450-
stage1_cxx,
451486
f"-DCMAKE_BUILD_TYPE={stage2_config}",
452487
"-DLLVM_ENABLE_ASSERTIONS=True",
453488
f"-DLLVM_LIT_ARGS={lit_args}",
454489
f"-DCMAKE_INSTALL_PREFIX=../{stage2_install}"
455-
] + (extra_stage2_cmake_args or extra_cmake_args)
490+
] + (extra_stage2_cmake_args or extra_cmake_args) \
491+
+ compiler_args
456492

457493
f.addStep(ShellCommand(name='cmake stage 2',
458494
command=cmake_cmd2,

0 commit comments

Comments
 (0)