Merge pull request #107 from TheBlueMatt/main

Support Building OSX binaries on Linux

Author: TheBlueMatt

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang
+          apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang wget
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
@@ -58,12 +58,61 @@ jobs:
             git diff --exit-code
           fi
 
+  check_macos:
+    runs-on: ubuntu-latest
+    # Ubuntu's version of rustc uses its own LLVM instead of being a real native package.
+    # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS.
+    container: debian:bookworm
+    env:
+      TOOLCHAIN: stable
+    steps:
+      - name: Install native Rust toolchain, Valgrind, and build utilitis
+        run: |
+          apt-get update
+          apt-get -y dist-upgrade
+          apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang wget rust-src
+      - name: Checkout source code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Sanity test bindings against Cargo.toml RL
+        working-directory: lightning-c-bindings
+        run: |
+          # Note that the version tags aren't checked into git
+          touch src/version.rs
+          RUSTFLAGS="--cfg=c_bindings" cargo check --features std
+      - name: Install cbindgen
+        run: |
+          git clone https://github.com/eqrion/cbindgen
+          cd cbindgen/
+          git checkout v0.20.0
+          cargo update -p indexmap --precise "1.6.2" --verbose
+          cargo install --locked --path .
+      - name: Checkout Rust-Lightning git
+        run: |
+          git clone https://github.com/rust-bitcoin/rust-lightning
+          cd rust-lightning
+          git checkout 0.0.115-bindings
+      - name: Fix Github Actions to not be broken
+        run: git config --global --add safe.directory /__w/ldk-c-bindings/ldk-c-bindings
+      - name: Fetch MacOS SDK
+        run: |
+          wget https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz
+          tar xvvf Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz
+      - name: Rebuild bindings, and check the sample app builds + links
+        run: |
+          # rust-src doesn't distribute the rustlib Cargo.lock, but an empty
+          # file seems to suffice to make `-Zbuild-std` happy.
+          touch /usr/lib/rustlib/src/rust/Cargo.lock
+          MACOS_SDK="$PWD/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" ./genbindings.sh ./rust-lightning true
+
   osx:
     strategy:
       matrix:
         include:
-          - platform: macos-10.15
           - platform: macos-11
+          - platform: macos-12
+          - platform: macos-13
     runs-on: ${{ matrix.platform }}
     env:
       TOOLCHAIN: stable

deterministic-build-wrappers/clang-lto-link-osx

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # We want to use rustc's -C linker-plugin-lto, but it passes several arguments
 # that are not understood by OSX clang/ld64.lld. Specifically, it passes
 # -plugin-opt* arguments to set optimization levels, which are not supported.

genbindings.sh

@@ -53,14 +53,20 @@ fi
 
 BASE_HOST_CFLAGS="$BASE_CFLAGS"
 
-if [ "$HOST_OSX" = "true" ]; then
+if [ "$MACOS_SDK" = "" -a "$HOST_OSX" = "true" ]; then
+	MACOS_SDK="$(xcrun --show-sdk-path)"
+	[ "$MACOS_SDK" = "" ] && exit 1
+fi
+
+if [ "$MACOS_SDK" != "" ]; then
 	export MACOSX_DEPLOYMENT_TARGET=10.9
-	LOCAL_CFLAGS="$LOCAL_CFLAGS --target=$HOST_PLATFORM -isysroot$(xcrun --show-sdk-path) -mmacosx-version-min=10.9"
-	BASE_HOST_CFLAGS="$BASE_HOST_CFLAGS --target=$HOST_PLATFORM -isysroot$(xcrun --show-sdk-path) -mmacosx-version-min=10.9"
-	# Targeting aarch64 appears to be supported only starting with Big Sur, so check it before use
-	clang -o /dev/null $BASE_HOST_CFLAGS --target=aarch64-apple-darwin -mcpu=apple-a14 genbindings_path_map_test_file.c &&
-	export CFLAGS_aarch64_apple_darwin="$BASE_HOST_CFLAGS --target=aarch64-apple-darwin -mcpu=apple-a14" ||
-	echo "WARNING: Can not build targeting aarch64-apple-darin. Upgrade to Big Sur or try upstream clang"
+	BASE_HOST_OSX_CFLAGS="$BASE_HOST_CFLAGS -isysroot$MACOS_SDK -mmacosx-version-min=10.9"
+	export CFLAGS_aarch64_apple_darwin="$BASE_HOST_OSX_CFLAGS --target=aarch64-apple-darwin -mcpu=apple-a14"
+	export CFLAGS_x86_64_apple_darwin="$BASE_HOST_OSX_CFLAGS --target=x86_64-apple-darwin -march=sandybridge -mtune=sandybridge"
+	if [ "$HOST_OSX" = "true" ]; then
+		LOCAL_CFLAGS="$LOCAL_CFLAGS --target=$HOST_PLATFORM -isysroot$MACOS_SDK -mmacosx-version-min=10.9"
+		BASE_HOST_CFLAGS="$BASE_HOST_OSX_CFLAGS --target=$HOST_PLATFORM"
+	fi
 fi
 
 rm genbindings_path_map_test_file.c
@@ -214,7 +220,7 @@ EOF
 cd lightning-c-bindings
 
 RUSTFLAGS="$RUSTFLAGS --cfg=test_mod_pointers" cargo build $CARGO_BUILD_ARGS
-if [ "$CFLAGS_aarch64_apple_darwin" != "" ]; then
+if [ "$CFLAGS_aarch64_apple_darwin" != "" -a "$HOST_OSX" = "true" ]; then
 	RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14" cargo build $CARGO_BUILD_ARGS --target aarch64-apple-darwin
 fi
 cbindgen -v --config cbindgen.toml -o include/lightning.h >/dev/null 2>&1
@@ -454,7 +460,7 @@ if [ "$HOST_PLATFORM" = "x86_64-unknown-linux-gnu" -o "$HOST_PLATFORM" = "x86_64
 			sed -i .bk 's/,"cdylib"]/]/g' Cargo.toml
 		fi
 
-		if [ "$CFLAGS_aarch64_apple_darwin" != "" ]; then
+		if [ "$CFLAGS_aarch64_apple_darwin" != "" -a "$HOST_OSX" = "true" ]; then
 			RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14" RUSTC_BOOTSTRAP=1 cargo rustc $CARGO_BUILD_ARGS --target aarch64-apple-darwin -v -- -Zsanitizer=address -Cforce-frame-pointers=yes || ( mv Cargo.toml.bk Cargo.toml; exit 1)
 		fi
 		RUSTFLAGS="$RUSTFLAGS --cfg=test_mod_pointers" RUSTC_BOOTSTRAP=1 cargo rustc $CARGO_BUILD_ARGS -v -- -Zsanitizer=address -Cforce-frame-pointers=yes || ( mv Cargo.toml.bk Cargo.toml; exit 1)
@@ -541,18 +547,27 @@ if [ "$CLANGPP" != "" -a "$LLD" != "" ]; then
 	# packaging than simply shipping the rustup binaries (eg Debian should Just Work
 	# here).
 	LINK_ARG_FLAGS="-C link-arg=-fuse-ld=$LLD"
-	if [ "$HOST_OSX" = "true" ]; then
-		export LDK_CLANG_PATH=$(which $CLANG)
+	export LDK_CLANG_PATH=$(which $CLANG)
+	if [ "$MACOS_SDK" != "" ]; then
 		export CLANG="$(pwd)/../deterministic-build-wrappers/clang-lto-link-osx"
-		for ARG in "CFLAGS_aarch64_apple_darwin"; do
+		for ARG in $CFLAGS_aarch64_apple_darwin; do
 			MANUAL_LINK_CFLAGS="$MANUAL_LINK_CFLAGS -C link-arg=$ARG"
 		done
 		export CFLAGS_aarch64_apple_darwin="$CFLAGS_aarch64_apple_darwin -O3 -fPIC -fembed-bitcode"
-		LINK_ARG_FLAGS="$LINK_ARG_FLAGS -C link-arg="-isysroot$(xcrun --show-sdk-path)" -C link-arg=-mmacosx-version-min=10.9"
-		RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14 -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $LINK_ARG_FLAGS -C link-arg=-mcpu=apple-a14" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target aarch64-apple-darwin
+		RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14 -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-mcpu=apple-a14" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target aarch64-apple-darwin -Zbuild-std=std,panic_abort
+		if [ "$HOST_OSX" != "true" ]; then
+			# If we're not on OSX but can build OSX binaries, build the x86_64 OSX release now
+			MANUAL_LINK_CFLAGS=""
+			for ARG in $CFLAGS_x86_64_apple_darwin; do
+				MANUAL_LINK_CFLAGS="$MANUAL_LINK_CFLAGS -C link-arg=$ARG"
+			done
+			export CFLAGS_x86_64_apple_darwin="$CFLAGS_x86_64_apple_darwin -O3 -fPIC -fembed-bitcode"
+			RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=sandybridge -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-mcpu=sandybridge -C link-arg=-mtune=sandybridge" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target x86_64-apple-darwin -Zbuild-std=std,panic_abort
+		fi
 	fi
 	# If we're on an M1 don't bother building X86 binaries
 	if [ "$HOST_PLATFORM" != "aarch64-apple-darwin" ]; then
+		[ "$HOST_OSX" != "true" ] && export CLANG="$LDK_CLANG_PATH"
 		export CFLAGS_$ENV_TARGET="$BASE_HOST_CFLAGS -O3 -fPIC -fembed-bitcode"
 		# Rust doesn't recognize CFLAGS changes, so we need to clean build artifacts
 		cargo clean --release