Skip to content

Commit 6f83b88

Browse files
reubenmillerreubenmiller
committed
refactor build script to support different build tools like clang and cargo-zigbuild
Signed-off-by: reubenmiller <reuben.d.miller@gmail.com>
1 parent ab72e5b commit 6f83b88

File tree

2 files changed

+175
-96
lines changed

2 files changed

+175
-96
lines changed

‎ci/build_scripts/build.sh

Lines changed: 170 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99
# References: https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u
1010
set -eo pipefail
1111

12+
# enable debugging by default in ci
13+
if [ "$CI" = "true" ]; then
14+
set -x
15+
fi
16+
1217
help() {
1318
cat <<EOF
1419
Compile and build the tedge components and linux packages.
@@ -22,15 +27,15 @@ Alternatively, if you would like to set a custom version (for development/testin
2227
the 'GIT_SEMVER' environment variable before calling this script.
2328
2429
Usage:
25-
$0 [ARCH]
30+
$0 [TARGET]
2631
2732
Args:
28-
ARCH RUST target architecture which can be a value listed from the command 'rustc --print target-list'
33+
TARGET RUST target architecture which can be a value listed from the command 'rustc --print target-list'
2934
If left blank then the TARGET will be set to the linux musl variant appropriate for your machine.
3035
For example, if building on MacOS M1, 'aarch64-unknown-linux-musl' will be selected, for linux x86_64,
3136
'x86_64-unknown-linux-musl' will be selected.
3237
33-
Example ARCH (target) values:
38+
Example TARGET values:
3439
3540
Linux MUSL variants
3641
* x86_64-unknown-linux-musl
@@ -44,17 +49,16 @@ Args:
4449
* armv7-unknown-linux-gnueabihf
4550
* arm-unknown-linux-gnueabihf
4651
47-
Linux GNU variants (controlling GNU libc version)
48-
# Be aware of the caveats: https://github.com/rust-cross/cargo-zigbuild?tab=readme-ov-file#specify-glibc-version
49-
* aarch64-unknown-linux-gnu.2.17
50-
* aarch64-unknown-linux-gnu.2.27
51-
5252
Apple
5353
* aarch64-apple-darwin
5454
* x86_64-apple-darwin
5555
5656
Flags:
5757
--help|-h Show this help
58+
--build-with <auto|zig|clang|native> Choose which tooling to use to build the binaries. If set to 'auto',
59+
then the build tool will be selected based on the binary
60+
--bin <name> Override which binary to build. By default the binaries form the package_list.sh will be used
61+
--glibc-version <version> GLIBC version to use when compiling for libc targets
5862
--skip-build Skip building the binaries and only package them (e.g. just create the linux packages)
5963
6064
Env:
@@ -67,6 +71,9 @@ Examples:
6771
$0 aarch64-unknown-linux-musl
6872
# Build for arm64 linux (musl)
6973
74+
$0 aarch64-unknown-linux-musl --build-with clang
75+
# Build for arm64 linux (musl) using clang (for linux only)
76+
7077
$0 x86_64-unknown-linux-musl
7178
# Build for x86_64 linux (musl)
7279
@@ -85,32 +92,55 @@ Examples:
8592
EOF
8693
}
8794

88-
ARCH=
89-
TARGET=()
90-
BUILD_OPTIONS=()
95+
# Load the release package list as $RELEASE_PACKAGES, BINARIES
96+
# shellcheck disable=SC1091
97+
source ./ci/package_list.sh
98+
99+
TARGET="${TARGET:-}"
100+
BUILD_WITH="${BUILD_WITH:-zig}"
101+
COMMON_BUILD_OPTIONS=(
102+
"--release"
103+
)
104+
TOOLCHAIN="${TOOLCHAIN:-+1.78}"
105+
# Note: Minimum version that is supported with riscv64gc-unknown-linux-gnu is 2.27
106+
GLIBC_VERSION="${GLIBC_VERSION:-2.17}"
107+
RISCV_GLIBC_VERSION="${RISCV_GLIBC_VERSION:-2.27}"
108+
OVERRIDE_BINARIES=()
109+
ARTIFACT_DIR="${ARTIFACT_DIR:-}"
110+
91111
BUILD=1
92-
INCLUDE_DEPRECATED_PACKAGES=0
93112

94113
REST_ARGS=()
95114
while [ $# -gt 0 ]
96115
do
97116
case "$1" in
117+
--build-with)
118+
BUILD_WITH="$2"
119+
shift
120+
;;
121+
--bin)
122+
OVERRIDE_BINARIES=( "$2" )
123+
shift
124+
;;
125+
--toolchain)
126+
TOOLCHAIN="+$2"
127+
shift
128+
;;
129+
--glibc-version)
130+
GLIBC_VERSION="$2"
131+
shift
132+
;;
98133
--skip-build)
99134
BUILD=0
100135
;;
101-
102-
--include-deprecated-packages)
103-
INCLUDE_DEPRECATED_PACKAGES=1
104-
;;
105-
--skip-deprecated-packages)
106-
INCLUDE_DEPRECATED_PACKAGES=0
136+
--artifact-dir)
137+
ARTIFACT_DIR="$2"
138+
shift
107139
;;
108-
109140
-h|--help)
110141
help
111142
exit 0
112143
;;
113-
114144
*)
115145
REST_ARGS+=("$1")
116146
;;
@@ -123,59 +153,36 @@ if [ ${#REST_ARGS[@]} -gt 0 ]; then
123153
set -- "${REST_ARGS[@]}"
124154
fi
125155

156+
if [ ${#OVERRIDE_BINARIES[@]} -gt 0 ]; then
157+
# Override the list of binaries to build
158+
BINARIES=("${OVERRIDE_BINARIES[@]}")
159+
fi
160+
126161
if [ $# -eq 1 ]; then
127-
ARCH="$1"
162+
TARGET="$1"
163+
fi
164+
165+
if [ -z "$TARGET" ]; then
166+
TARGET=$(./ci/build_scripts/detect_target.sh)
128167
fi
129168

130169
# Set version from scm
131170
# Run before installing any dependencies so that it
132171
# can be called from other tools without requiring cargo
133172
# shellcheck disable=SC1091
134-
. ./ci/build_scripts/version.sh
135-
136-
if [ -z "$ARCH" ]; then
137-
# If no target has been given, choose the target triple based on the
138-
# host's architecture, however use the musl builds by default!
139-
HOST_ARCH="$(uname -m || true)"
140-
case "$HOST_ARCH" in
141-
x86_64*|amd64*)
142-
ARCH=x86_64-unknown-linux-musl
143-
;;
144-
145-
aarch64|arm64)
146-
ARCH=aarch64-unknown-linux-musl
147-
;;
148-
149-
armv7*)
150-
ARCH=armv7-unknown-linux-musleabihf
151-
;;
152-
153-
armv6*)
154-
ARCH=arm-unknown-linux-musleabi
155-
;;
156-
esac
157-
fi
158-
173+
source ./ci/build_scripts/version.sh
159174

160-
# Load the release package list as $RELEASE_PACKAGES, $DEPRECATED_PACKAGES
161-
# shellcheck disable=SC1091
162-
source ./ci/package_list.sh
163-
164-
# Note: cargo-zigbuild supports specifying the specific gnu libc version to use
165-
# but Rust doesn't, so the target needs to be normalized to match the target without
166-
# the gnu libc version information
167-
ZIG_TARGET="$ARCH"
168-
ARCH=$(echo "$ZIG_TARGET" | cut -d. -f1)
169-
170-
# build release for target
171-
# GIT_SEMVER should be referenced in the build.rs scripts
172-
if [ "$BUILD" = 1 ]; then
173-
# Install stable toolchain if missing
175+
install_rust() {
176+
# Install toolchain if missing
174177
if command -V rustup >/dev/null 2>&1; then
175-
rustup toolchain install stable --no-self-update
178+
rustup toolchain install "${TOOLCHAIN//+/}" --no-self-update
176179
fi
177-
# Use zig to build as it is provides better cross compiling support
178-
cargo +stable install cargo-zigbuild --version ">=0.17.3"
180+
}
181+
182+
install_zig_tools() {
183+
# zig provides better cross compiling support
184+
# shellcheck disable=SC2086
185+
cargo $TOOLCHAIN install cargo-zigbuild --version ">=0.17.3"
179186

180187
# Allow users to install zig by other package managers
181188
if ! zig --help &>/dev/null; then
@@ -186,43 +193,115 @@ if [ "$BUILD" = 1 ]; then
186193

187194
# Display zig version to help with debugging
188195
echo "zig version: $(zig version 2>/dev/null || python3 -m ziglang version 2>/dev/null ||:)"
196+
}
189197

190-
if [ -n "$ARCH" ]; then
191-
echo "Using target: $ARCH (zig target=$ZIG_TARGET)"
192-
TARGET+=("--target=$ZIG_TARGET")
193-
rustup target add "$ARCH"
194-
else
195-
# Note: This will build the artifacts under target/release and not target/<triple>/release !
196-
HOST_TARGET=$(rustc --version --verbose | grep host: | cut -d' ' -f2)
197-
echo "Using host target: $HOST_TARGET"
198-
fi
198+
build() {
199+
build_tool="$1"
200+
shift
201+
case "$build_tool" in
202+
zig|ziglang|cargo-zigbuild)
203+
# shellcheck disable=SC2086
204+
cargo-zigbuild $TOOLCHAIN zigbuild "$@"
205+
;;
206+
clang)
207+
# shellcheck disable=SC2086
208+
mk/cargo.sh $TOOLCHAIN build "$@"
209+
;;
210+
native|*)
211+
# shellcheck disable=SC2086
212+
cargo $TOOLCHAIN build "$@"
213+
;;
214+
esac
215+
}
199216

200-
# Custom options for different targets
201-
case "$ARCH" in
217+
get_build_tool_for_binary() {
218+
# Different binaries have different requirements / build dependencies
219+
# which influence which build tools can be used.
220+
# Previously tedge has been using clang to build the binaries, so clang
221+
# should still be preferred to reduce risk of unexpected differences between
222+
# different compiler optimizations (not sure if this is true, but less changes are generally safer)
223+
binary="$1"
224+
case "$binary" in
225+
tedge-p11-server)
226+
echo "zig"
227+
;;
228+
*)
229+
echo "clang"
230+
esac
231+
}
232+
233+
get_target_for_binary() {
234+
binary_name="$1"
235+
target="$2"
236+
case "$binary_name" in
237+
tedge-p11-server)
238+
# requires gnu target as loading .so files requires to be dynamically compiled
239+
# This can return the same output, but this is fine as apple targets support
240+
# loading by default
241+
echo "${target//musl/gnu}"
242+
;;
202243
*)
203-
BUILD_OPTIONS+=(
204-
--release
205-
)
244+
echo "$target"
206245
;;
207246
esac
247+
}
208248

209-
cargo zigbuild "${TARGET[@]}" "${BUILD_OPTIONS[@]}"
249+
if [ -z "$ARTIFACT_DIR" ]; then
250+
ARTIFACT_DIR="target/$TARGET/release"
210251
fi
252+
mkdir -p "$ARTIFACT_DIR"
211253

212-
# Create release packages
213-
OUTPUT_DIR="target/$ARCH/packages"
254+
# build release for target
255+
# GIT_SEMVER should be referenced in the build.rs scripts
256+
if [ "$BUILD" = 1 ] && [ ${#BINARIES[@]} -gt 0 ]; then
257+
install_rust
258+
259+
for name in "${BINARIES[@]}"; do
260+
BINARY_TARGET=$(get_target_for_binary "$name" "$TARGET")
261+
# shellcheck disable=SC2086
262+
rustup $TOOLCHAIN target add "$BINARY_TARGET"
263+
BUILD_DIR="target/$BINARY_TARGET/release"
264+
265+
# Each binary should have its preferred build tool (unless if the user overrides this)
266+
BUILD_TOOL=$(get_build_tool_for_binary "$name")
267+
if [ -n "$BUILD_WITH" ] && [ "$BUILD_WITH" != "auto" ]; then
268+
BUILD_TOOL="$BUILD_WITH"
269+
fi
214270

215-
# Remove deprecated debian folder to avoid confusion with newly built linux packages
216-
if [ -d "target/$ARCH/debian" ]; then
217-
echo "Removing deprecated debian folder created by cargo-deb: target/$ARCH/debian" >&2
218-
rm -rf "target/$ARCH/debian"
271+
case "$BUILD_TOOL" in
272+
zig)
273+
install_zig_tools
274+
275+
case "$BINARY_TARGET" in
276+
riscv64gc-unknown-linux-gnu)
277+
# riscv is a newer processor so the minimum glibc version is higher than for other targets
278+
if [ -n "$RISCV_GLIBC_VERSION" ]; then
279+
BINARY_TARGET="${BINARY_TARGET}.${RISCV_GLIBC_VERSION}"
280+
fi
281+
;;
282+
*gnu*)
283+
if [ -n "$GLIBC_VERSION" ]; then
284+
BINARY_TARGET="${BINARY_TARGET}.${GLIBC_VERSION}"
285+
fi
286+
;;
287+
esac
288+
;;
289+
clang)
290+
# shellcheck disable=SC2086
291+
./mk/install-build-tools.sh $TOOLCHAIN --target="$BINARY_TARGET"
292+
;;
293+
*)
294+
;;
295+
esac
296+
297+
build "$BUILD_TOOL" --target="$BINARY_TARGET" "${COMMON_BUILD_OPTIONS[@]}" --bin "$name"
298+
if [ "$BUILD_DIR" != "$ARTIFACT_DIR" ]; then
299+
cp "$BUILD_DIR/$name" "$ARTIFACT_DIR/"
300+
fi
301+
done
219302
fi
220303

304+
# Create release packages (e.g. linux packages like rpm, deb, apk etc.)
305+
OUTPUT_DIR="$(dirname "$ARTIFACT_DIR")/packages"
221306
PACKAGES=( "${RELEASE_PACKAGES[@]}" )
222-
if [ "$INCLUDE_DEPRECATED_PACKAGES" = "1" ]; then
223-
PACKAGES+=(
224-
"${DEPRECATED_PACKAGES[@]}"
225-
)
226-
fi
227-
228-
./ci/build_scripts/package.sh build "$ARCH" "${PACKAGES[@]}" --output "$OUTPUT_DIR"
307+
./ci/build_scripts/package.sh build "$TARGET" "${PACKAGES[@]}" --output "$OUTPUT_DIR"

‎ci/package_list.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ RELEASE_PACKAGES=(
1212
)
1313
export RELEASE_PACKAGES
1414

15-
# Deprecated packages are still built but not explicitly tested
16-
# This allows users to still access the packages if needed, however
17-
# it is only reserved for packages with a more public facing API
18-
DEPRECATED_PACKAGES=()
19-
export DEPRECATED_PACKAGES
15+
# List of binaries which should be built
16+
BINARIES=(
17+
tedge
18+
)
19+
export BINARIES

0 commit comments

Comments
 (0)