From b109eebbf86ba531e87cbb071785d3331e747d1b Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 21 May 2025 00:36:25 +0100 Subject: [PATCH 1/5] Merge docker images with extra build-arg not the best name, but at it helps me identify urls discrepancies maybe. Signed-off-by: clux --- .github/workflows/nightly.yml | 3 + .github/workflows/stable.yml | 14 ++--- Dockerfile.arm64 => Dockerfile | 19 +++++-- Dockerfile.x86_64 | 101 --------------------------------- justfile | 14 ++--- test.sh | 1 - 6 files changed, 29 insertions(+), 123 deletions(-) rename Dockerfile.arm64 => Dockerfile (81%) delete mode 100644 Dockerfile.x86_64 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9419787..5b53193 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -28,10 +28,12 @@ jobs: - platform: linux/amd64 dockerfile: Dockerfile.x86_64 arch: amd64 + aarch: x86_64 target_dir: x86_64-unknown-linux-musl - platform: linux/arm64 dockerfile: Dockerfile.arm64 arch: arm64 + aarch: aarch64 target_dir: aarch64-unknown-linux-musl steps: - uses: actions/checkout@v4 @@ -73,6 +75,7 @@ jobs: tags: rustmusl-temp build-args: | CHANNEL=nightly + AARCH=${{ matrix.aarch }} - name: Run tests shell: bash diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index c2eaa14..26eb40d 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -26,12 +26,14 @@ jobs: platform: [linux/amd64, linux/arm64] include: - platform: linux/amd64 - dockerfile: Dockerfile.x86_64 + dockerfile: Dockerfile arch: amd64 + aarch: x86_64 target_dir: x86_64-unknown-linux-musl - platform: linux/arm64 - dockerfile: Dockerfile.arm64 + dockerfile: Dockerfile arch: arm64 + aarch: aarch64 target_dir: aarch64-unknown-linux-musl steps: - uses: actions/checkout@v4 @@ -85,6 +87,7 @@ jobs: tags: rustmusl-temp build-args: | CHANNEL=stable + AARCH=${{ matrix.aarch }} if: ${{ steps.stablecheck.outputs.BUILD }} - name: Run tests @@ -189,9 +192,4 @@ jobs: docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:$tag \ ${{ env.REGISTRY_IMAGE }}:amd64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} \ ${{ env.REGISTRY_IMAGE }}:arm64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} - done - - - name: Inspect image - if: ${{ steps.stablecheck.outputs.BUILD }} - run: | - docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest diff --git a/Dockerfile.arm64 b/Dockerfile similarity index 81% rename from Dockerfile.arm64 rename to Dockerfile index af3c1eb..4e61a73 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile @@ -1,6 +1,11 @@ # syntax=docker/dockerfile:1 FROM ubuntu:noble LABEL maintainer="Eirik Albrigtsen " +LABEL org.opencontainers.image.create="$(date --utc --iso-8601=seconds)" +LABEL org.opencontainers.image.documentation="https://github.com/clux/muslrust" +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.url="https://github.com/clux/muslrust" +LABEL org.opencontainers.image.description="Docker environment for building musl based static rust binaries" # Required packages: # - musl-dev, musl-tools - the musl toolchain @@ -30,17 +35,18 @@ RUN apt-get update && apt-get install -y \ --no-install-recommends && \ rm -rf /var/lib/apt/lists/* +ARG AARCH # Install rust using rustup ARG CHANNEL ENV RUSTUP_VER="1.27.1" \ - RUST_ARCH="aarch64-unknown-linux-gnu" \ + RUST_ARCH="${AARCH}-unknown-linux-gnu" \ CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUN curl "https://static.rust-lang.org/rustup/archive/${RUSTUP_VER}/${RUST_ARCH}/rustup-init" -o rustup-init && \ chmod +x rustup-init && \ ./rustup-init -y --default-toolchain ${CHANNEL} --profile minimal --no-modify-path && \ rm rustup-init && \ - ~/.cargo/bin/rustup target add aarch64-unknown-linux-musl + ~/.cargo/bin/rustup target add ${AARCH}-unknown-linux-musl # Allow non-root access to cargo RUN chmod a+X /root @@ -59,13 +65,13 @@ ENV ZLIB_VER="1.3.1" \ # Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features) RUN cd /tmp && \ - curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-aarch_64.zip -o protoc.zip && \ + curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-${AARCH}.zip -o protoc.zip && \ unzip protoc.zip && \ cp bin/protoc /usr/bin/protoc && \ rm -rf * # Install prebuilt sccache based on platform -RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-aarch64-unknown-linux-musl.tar.gz | tar xz && \ +RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-${AARCH}-unknown-linux-musl.tar.gz | tar xz && \ mv sccache-v${SCCACHE_VER}-*-unknown-linux-musl/sccache /usr/local/bin/ && \ chmod +x /usr/local/bin/sccache && \ rm -rf sccache-v${SCCACHE_VER}-*-unknown-linux-musl @@ -88,13 +94,16 @@ RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | t ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ RUSTUP_HOME=/root/.rustup \ - CARGO_BUILD_TARGET=aarch64-unknown-linux-musl \ + CARGO_BUILD_TARGET=${AARCH}-unknown-linux-musl \ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld -Ctarget-feature=+crt-static" \ PKG_CONFIG_ALLOW_CROSS=true \ PKG_CONFIG_ALL_STATIC=true \ PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ + PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ PG_CONFIG_AARCH64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ + # Rust libz-sys support LIBZ_SYS_STATIC=1 \ + ZLIB_STATIC=1 \ DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 deleted file mode 100644 index aae5da5..0000000 --- a/Dockerfile.x86_64 +++ /dev/null @@ -1,101 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM ubuntu:noble -LABEL maintainer="Eirik Albrigtsen " - -# Required packages: -# - musl-dev, musl-tools - the musl toolchain -# - curl, g++, make, pkgconf, cmake - for fetching and building third party libs -# - ca-certificates - openssl + curl + peer verification of downloads -# - git - cargo builds in user projects -# - file - needed by rustup.sh install -# - automake autoconf libtool - support crates building C deps as part cargo build -# NB: does not include cmake atm -RUN apt-get update && apt-get install -y \ - musl-dev \ - musl-tools \ - file \ - git \ - openssh-client \ - make \ - cmake \ - g++ \ - curl \ - pkgconf \ - ca-certificates \ - automake \ - autoconf \ - libtool \ - libprotobuf-dev \ - unzip \ - --no-install-recommends && \ - rm -rf /var/lib/apt/lists/* - -# Install rust using rustup -ARG CHANNEL -ENV RUSTUP_VER="1.27.1" \ - RUST_ARCH="x86_64-unknown-linux-gnu" \ - CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse - -RUN curl "https://static.rust-lang.org/rustup/archive/${RUSTUP_VER}/${RUST_ARCH}/rustup-init" -o rustup-init && \ - chmod +x rustup-init && \ - ./rustup-init -y --default-toolchain ${CHANNEL} --profile minimal --no-modify-path && \ - rm rustup-init && \ - ~/.cargo/bin/rustup target add x86_64-unknown-linux-musl - -# Allow non-root access to cargo -RUN chmod a+X /root - -# Convenience list of versions and variables for compilation later on -# This helps continuing manually if anything breaks. -ENV ZLIB_VER="1.3.1" \ - SQLITE_VER="3490100" \ - PROTOBUF_VER="29.2" \ - SCCACHE_VER="0.9.1" \ - CC=musl-gcc \ - PREFIX=/musl \ - PATH=/usr/local/bin:/root/.cargo/bin:$PATH \ - PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ - LD_LIBRARY_PATH=$PREFIX - -# Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features) -RUN cd /tmp && \ - curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-x86_64.zip -o protoc.zip && \ - unzip protoc.zip && \ - cp bin/protoc /usr/bin/protoc && \ - rm -rf * - -# Install prebuilt sccache based on platform -RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VER}/sccache-v${SCCACHE_VER}-x86_64-unknown-linux-musl.tar.gz | tar xz && \ - mv sccache-v${SCCACHE_VER}-*-unknown-linux-musl/sccache /usr/local/bin/ && \ - chmod +x /usr/local/bin/sccache && \ - rm -rf sccache-v${SCCACHE_VER}-*-unknown-linux-musl - -# Build zlib (used in pq) -RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ - cd zlib-$ZLIB_VER && \ - CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ - make -j$(nproc) && make install && \ - cd .. && rm -rf zlib-$ZLIB_VER - -# Build libsqlite3 using same configuration as the alpine linux main/sqlite package -RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ - cd sqlite-autoconf-$SQLITE_VER && \ - CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \ - CC="musl-gcc -fPIC -pie" \ - ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --disable-shared && \ - make && make install && \ - cd .. && rm -rf sqlite-autoconf-$SQLITE_VER - -ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ - RUSTUP_HOME=/root/.rustup \ - CARGO_BUILD_TARGET=x86_64-unknown-linux-musl \ - PKG_CONFIG_ALLOW_CROSS=true \ - PKG_CONFIG_ALL_STATIC=true \ - PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ - PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - LIBZ_SYS_STATIC=1 \ - DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC - -# Allow ditching the -w /volume flag to docker run -WORKDIR /volume diff --git a/justfile b/justfile index 39e44db..eb5e5fc 100644 --- a/justfile +++ b/justfile @@ -3,16 +3,16 @@ default: @just --list --unsorted --color=always -_build channel ar platform ext: - docker build --build-arg CHANNEL="{{channel}}" --build-arg AR="{{ar}}" --platform="{{platform}}" -t rustmusl-temp . -f Dockerfile.{{ext}} +_build channel platform aarch: + docker build --build-arg CHANNEL="{{channel}}" --build-arg AARCH="{{aarch}}" --platform="{{platform}}" -t rustmusl-temp . -f Dockerfile # Build the stable x86 container -build-stable-amd: (_build "stable" "amd64" "linux/amd64" "x86_64") +build-stable-amd: (_build "stable" "linux/amd64" "x86_64") # Build the nightly x86 container -build-nightly-amd: (_build "nightly" "amd64" "linux/amd64" "x86_64") +build-nightly-amd: (_build "nightly" "linux/amd64" "x86_64") # Build the stable arm container -build-stable-arm: (_build "stable" "arm64" "linux/arm64" "arm64") +build-stable-arm: (_build "stable" "linux/arm64" "aarch64") # Build the nightly arm container -build-nightly-arm: (_build "nightly" "arm64" "linux/arm64" "arm64") +build-nightly-arm: (_build "nightly" "linux/arm64" "aarch64") # Shell into the built container run: @@ -36,13 +36,11 @@ _t_linux_x86_64 crate: #!/bin/bash export PLATFORM="linux/amd64" export TARGET_DIR="x86_64-unknown-linux-musl" - export AR="amd64" ./test.sh {{crate}} _t_macos_aarch64 crate: #!/bin/bash export PLATFORM="linux/arm64" export TARGET_DIR="aarch64-unknown-linux-musl" - export AR="arm64" ./test.sh {{crate}} # Test all crates against built container locally diff --git a/test.sh b/test.sh index a1ff77e..65b4d1f 100755 --- a/test.sh +++ b/test.sh @@ -11,7 +11,6 @@ docker_build() { -v "$PWD/test/${crate}:/volume" \ -v cargo-cache:/root/.cargo/registry \ -e RUST_BACKTRACE=1 \ - -e AR=ar \ --platform "${PLATFORM}" \ rustmusl-temp \ cargo build From a9b4067f906c665ca4fc70b1098cffe12833d8f7 Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 21 May 2025 00:39:32 +0100 Subject: [PATCH 2/5] fix accidental edit Signed-off-by: clux --- .github/workflows/stable.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index 26eb40d..10ab96f 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -192,4 +192,9 @@ jobs: docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:$tag \ ${{ env.REGISTRY_IMAGE }}:amd64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} \ ${{ env.REGISTRY_IMAGE }}:arm64-${RUST_VER}-${RUST_CHANNEL}-${RUST_DATE} - docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest + done + + - name: Inspect image + if: ${{ steps.stablecheck.outputs.BUILD }} + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:latest From 35772ccb08150436f57488ceed527b553dba16bd Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 21 May 2025 00:41:00 +0100 Subject: [PATCH 3/5] fix dockerfile name Signed-off-by: clux --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5b53193..c3e1137 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,12 +26,12 @@ jobs: platform: [linux/amd64, linux/arm64] include: - platform: linux/amd64 - dockerfile: Dockerfile.x86_64 + dockerfile: Dockerfile arch: amd64 aarch: x86_64 target_dir: x86_64-unknown-linux-musl - platform: linux/arm64 - dockerfile: Dockerfile.arm64 + dockerfile: Dockerfile arch: arm64 aarch: aarch64 target_dir: aarch64-unknown-linux-musl From 6ed3c2d05c48e1e9d49d2b01fd5a48ba29df6c1a Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 21 May 2025 10:39:41 +0100 Subject: [PATCH 4/5] horrible bash ternary to special case pb url Signed-off-by: clux --- Dockerfile | 7 ++++--- justfile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4e61a73..69ae404 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ RUN apt-get update && apt-get install -y \ --no-install-recommends && \ rm -rf /var/lib/apt/lists/* +# Common arg for arch used in urls and triples ARG AARCH # Install rust using rustup ARG CHANNEL @@ -55,7 +56,7 @@ RUN chmod a+X /root # This helps continuing manually if anything breaks. ENV ZLIB_VER="1.3.1" \ SQLITE_VER="3490100" \ - PROTOBUF_VER="29.2" \ + PROTOBUF_VER="31.0" \ SCCACHE_VER="0.9.1" \ CC=musl-gcc \ PREFIX=/musl \ @@ -65,7 +66,7 @@ ENV ZLIB_VER="1.3.1" \ # Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features) RUN cd /tmp && \ - curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-${AARCH}.zip -o protoc.zip && \ + curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-$([ "$AARCH" = "aarch64"] && echo "aarch_64" || echo "$AARCH").zip -o protoc.zip && \ unzip protoc.zip && \ cp bin/protoc /usr/bin/protoc && \ rm -rf * @@ -76,7 +77,7 @@ RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VE chmod +x /usr/local/bin/sccache && \ rm -rf sccache-v${SCCACHE_VER}-*-unknown-linux-musl -# Build zlib (used in pq) +# Build zlib RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ cd zlib-$ZLIB_VER && \ CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ diff --git a/justfile b/justfile index eb5e5fc..26164d0 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,7 @@ default: @just --list --unsorted --color=always _build channel platform aarch: - docker build --build-arg CHANNEL="{{channel}}" --build-arg AARCH="{{aarch}}" --platform="{{platform}}" -t rustmusl-temp . -f Dockerfile + docker build --build-arg CHANNEL="{{channel}}" --build-arg AARCH="{{aarch}}" --platform="{{platform}}" -t rustmusl-temp . -f Dockerfile --debug # Build the stable x86 container build-stable-amd: (_build "stable" "linux/amd64" "x86_64") # Build the nightly x86 container From 8c3fe21b0767fb9feb8892a131dae5f18d05757b Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 21 May 2025 11:10:32 +0100 Subject: [PATCH 5/5] fix syntax in horrible ternary Signed-off-by: clux --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 69ae404..ed5bc7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ ENV ZLIB_VER="1.3.1" \ # Install a more recent release of protoc (protobuf-compiler in jammy is 4 years old and misses some features) RUN cd /tmp && \ - curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-$([ "$AARCH" = "aarch64"] && echo "aarch_64" || echo "$AARCH").zip -o protoc.zip && \ + curl -sSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VER}/protoc-${PROTOBUF_VER}-linux-$([ "$AARCH" = "aarch64" ] && echo "aarch_64" || echo "$AARCH").zip -o protoc.zip && \ unzip protoc.zip && \ cp bin/protoc /usr/bin/protoc && \ rm -rf *