From 180f86fe414b7d826ad6ace3705899bb34f1f22d Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Fri, 11 Apr 2025 10:40:32 -0700 Subject: [PATCH 1/2] Fix s390x dynamic loader symlink This change resolves an issue with the s390x dynamic loader symlink. On s390x UBI 8, the symlink points to `/usr/lib64`. This directory is not included in the Calico base image, resulting a dangling symlink and preventing programs from loading. This update corrects the symlink and added a test to validate the minimum C runtime. It also fixes semvalidator multi-arch builds. --- cmd/Makefile | 3 ++- images/calico-base/Dockerfile | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/Makefile b/cmd/Makefile index 628f1a99..797503cb 100644 --- a/cmd/Makefile +++ b/cmd/Makefile @@ -6,7 +6,8 @@ build: semvalidator-build-$(ARCH) .PHONY: semvalidator-build-$(ARCH) semvalidator-build-$(ARCH): semvalidator/main.go - CGO_ENABLED=0 go build -o bin/semvalidator-$(ARCH) -v -buildvcs=false -ldflags "-s -w" semvalidator/main.go + CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) \ + go build -o bin/semvalidator-$(ARCH) -v -buildvcs=false -ldflags "-s -w" semvalidator/main.go .PHONY: clean clean: diff --git a/images/calico-base/Dockerfile b/images/calico-base/Dockerfile index 7531a2b6..4a5377f9 100644 --- a/images/calico-base/Dockerfile +++ b/images/calico-base/Dockerfile @@ -3,6 +3,7 @@ FROM --platform=linux/amd64 calico/qemu-user-static:latest AS qemu FROM registry.access.redhat.com/ubi8/ubi-minimal:latest AS ubi ARG LDSONAME +ARG TARGETARCH COPY --from=qemu /usr/bin/qemu-*-static /usr/bin/ @@ -13,12 +14,16 @@ RUN microdnf upgrade -y RUN mkdir -p /rootfs/lib64 /rootfs/etc # Copy dynamic loader and symbolic links. -# Note: The dynamic loader name and links might be different in a future release. +# For s390x architecture, modify the /lib/${LDSONAME} symlink to ../lib64/${LDSONAME} +# instead of /usr/lib64 as the /usr/lib64 directory is not included in our base. RUN cp /lib64/ld-2.28.so /rootfs/lib64/ld-2.28.so RUN set -eux; \ cp -a /lib64/${LDSONAME} /rootfs/lib64/${LDSONAME}; \ if [ -f /lib/${LDSONAME} ]; then \ mkdir -p /rootfs/lib && cp -a /lib/${LDSONAME} /rootfs/lib/${LDSONAME}; \ + if [ "${TARGETARCH}" = "s390x" ]; then \ + ln -sf ../lib64/${LDSONAME} /rootfs/lib/${LDSONAME}; \ + fi \ fi # Required external C dependencies for CGO builds. @@ -42,6 +47,10 @@ FROM scratch AS source COPY --from=ubi /rootfs / +# Verify if glibc can be properly loaded. +# This check ensures that the dynamic loader and symbolic links are copied correctly. +RUN ["/lib64/libc.so.6"] + # tmp.tar has a /tmp with the correct permissions 01777. ADD tmp.tar / From d1b34d0f95d71a6fef877adad2cca76a1ba0b9cc Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Mon, 17 Feb 2025 09:24:02 -0800 Subject: [PATCH 2/2] Register and install qemu from tonistiigi/binfmt --- .gitignore | 1 - .semaphore/promotions/qemu-user-static.yml | 34 ---------------------- .semaphore/semaphore.yml | 4 --- Makefile.common | 2 +- README.md | 4 +-- images/Makefile | 27 +++-------------- images/calico-base/Dockerfile | 4 --- images/calico-go-build/Dockerfile | 6 ---- images/qemu-user-static/Dockerfile | 14 --------- 9 files changed, 7 insertions(+), 89 deletions(-) delete mode 100644 .semaphore/promotions/qemu-user-static.yml delete mode 100644 images/qemu-user-static/Dockerfile diff --git a/.gitignore b/.gitignore index e53642de..d46ced73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ cmd/bin -images/.qemu-user-static.created images/calico-go-build/bin diff --git a/.semaphore/promotions/qemu-user-static.yml b/.semaphore/promotions/qemu-user-static.yml deleted file mode 100644 index 8f70ac83..00000000 --- a/.semaphore/promotions/qemu-user-static.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: v1.0 -name: Publish calico/qemu-user-static image -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 30 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - -blocks: - - name: Publish calico/qemu-user-static amd64 image - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d'" - task: - env_vars: - - name: BRANCH_NAME - value: latest - jobs: - - name: Linux amd64 - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images qemu-user-static-cd CONFIRM=true; fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index fe9ee9dd..ebfad530 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -40,10 +40,6 @@ promotions: pipeline_file: promotions/calico-go-build.yml auto_promote: when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d'" - - name: Publish calico/qemu-user-static image - pipeline_file: promotions/qemu-user-static.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d'" blocks: - name: calico/go-build image diff --git a/Makefile.common b/Makefile.common index c0138586..b83be6f4 100644 --- a/Makefile.common +++ b/Makefile.common @@ -75,7 +75,7 @@ endif # This is only needed when running non-native binaries. register: ifneq ($(BUILDARCH),$(ARCH)) - docker run --rm --privileged multiarch/qemu-user-static:register || true + docker run --privileged --rm tonistiigi/binfmt --install all || true endif # If this is a release, also tag and push additional images. diff --git a/README.md b/README.md index 319185fc..f3666292 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ For example, if you registered the `s390x` emulator at `/usr/bin/qemu-s390x-stat To register emulators, we run: ```bash -docker run -it --rm --privileged multiarch/qemu-user-static:register +docker run --privileged --rm tonistiigi/binfmt --install all ``` or simply @@ -69,7 +69,7 @@ To _run_ a binary from a different architecture, you need to use `binfmt` and `q Register `qemu-*-static` for all supported processors except the current one using the following command: ```bash -docker run --rm --privileged multiarch/qemu-user-static:register +docker run --privileged --rm tonistiigi/binfmt --install all ``` If a cross built binary is executed in the go-build container qemu-static will automatically be used. diff --git a/images/Makefile b/images/Makefile index 29ed4055..ea5ecd8d 100644 --- a/images/Makefile +++ b/images/Makefile @@ -6,28 +6,11 @@ VERSION_TAG ?= latest CALICO_BASE ?= base CALICO_GO_BUILD ?= go-build -QEMU_USER_STATIC ?= $(DEV_REGISTRIES)/qemu-user-static -QEMU_USER_STATIC_IMAGE_CREATED = .qemu-user-static.created - .PHONY: image -image: qemu-user-static-image calico-base-image calico-go-build-image +image: calico-base-image calico-go-build-image .PHONY: image-all -image-all: qemu-user-static-image calico-base-image-all calico-go-build-image-all - -# Holder image for all qemu-*-static binaries Calico supports. -# It only builds for linux/amd64 platform. -.PHONY: qemu-user-static-image -qemu-user-static-image: $(QEMU_USER_STATIC_IMAGE_CREATED) -$(QEMU_USER_STATIC_IMAGE_CREATED): - docker buildx build $(DOCKER_PROGRESS) --load --platform=linux/amd64 --pull \ - -t $(QEMU_USER_STATIC):latest \ - -f qemu-user-static/Dockerfile qemu-user-static/ - touch $@ - -.PHONY: qemu-user-static-cd -qemu-user-static-cd: qemu-user-static-image - docker push $(QEMU_USER_STATIC):latest +image-all: calico-base-image-all calico-go-build-image-all # Base image for all calico components. @@ -44,7 +27,7 @@ else ifeq ($(ARCH),s390) endif .PHONY: calico-base-image -calico-base-image: register qemu-user-static-image +calico-base-image: register $(DOCKER_BUILD) --build-arg LDSONAME=$(LDSONAME) -t $(CALICO_BASE):latest-$(ARCH) -f calico-base/Dockerfile calico-base/ $(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=latest @@ -65,7 +48,7 @@ build: cp ../cmd/bin/semvalidator-$(ARCH) calico-go-build/bin/semvalidator-$(ARCH) .PHONY: calico-go-build-image -calico-go-build-image: register qemu-user-static-image build +calico-go-build-image: register build $(DOCKER_BUILD) -t $(CALICO_GO_BUILD):latest-$(ARCH) -f calico-go-build/Dockerfile calico-go-build/ $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(VERSION_TAG) @@ -84,8 +67,6 @@ push-calico-go-build-manifests: var-require-one-of-CONFIRM-DRYRUN var-require-al .PHONY: clean clean: - rm -f $(QEMU_USER_STATIC_IMAGE_CREATED) rm -fr calico-go-build/bin -docker image rm -f $$(docker images $(CALICO_BASE) -a -q) -docker image rm -f $$(docker images $(CALICO_GO_BUILD) -a -q) - -docker image rm -f $$(docker images $(QEMU_USER_STATIC) -a -q) diff --git a/images/calico-base/Dockerfile b/images/calico-base/Dockerfile index 4a5377f9..4d87bb48 100644 --- a/images/calico-base/Dockerfile +++ b/images/calico-base/Dockerfile @@ -1,12 +1,8 @@ -FROM --platform=linux/amd64 calico/qemu-user-static:latest AS qemu - FROM registry.access.redhat.com/ubi8/ubi-minimal:latest AS ubi ARG LDSONAME ARG TARGETARCH -COPY --from=qemu /usr/bin/qemu-*-static /usr/bin/ - RUN microdnf upgrade -y # Prepare a rootfs for necessary files from UBI. diff --git a/images/calico-go-build/Dockerfile b/images/calico-go-build/Dockerfile index a3b5eab5..f196d10b 100644 --- a/images/calico-go-build/Dockerfile +++ b/images/calico-go-build/Dockerfile @@ -2,8 +2,6 @@ ARG TARGETARCH=${TARGETARCH} FROM calico/bpftool:v7.4.0 AS bpftool -FROM --platform=amd64 calico/qemu-user-static:latest AS qemu - FROM registry.access.redhat.com/ubi8/ubi:latest AS ubi ARG TARGETARCH @@ -16,10 +14,6 @@ ARG YQ_VERSION=v4.45.1 ENV PATH=/usr/local/go/bin:$PATH -# Enable non-native runs on amd64 architecture hosts -# Supported qemu-user-static arch files are copied in Makefile `download-qemu` target -COPY --from=qemu /usr/bin/qemu-*-static /usr/bin - # Install system dependencies RUN dnf upgrade -y && dnf install -y \ autoconf \ diff --git a/images/qemu-user-static/Dockerfile b/images/qemu-user-static/Dockerfile deleted file mode 100644 index 2f8dca26..00000000 --- a/images/qemu-user-static/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM fedora:latest AS qemu - -RUN dnf install -y qemu-user-static - -FROM scratch AS source - -COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static -COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin/qemu-ppc64le-static -COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin/qemu-s390x-static -COPY --from=qemu /usr/bin/qemu-x86_64-static /usr/bin/qemu-x86_64-static - -FROM scratch - -COPY --from=source / /