Support multi-arch builds with separate workers

.github/workflows/wkdev-sdk.yml

@@ -10,8 +10,8 @@ defaults:
   run:
     shell: bash
 jobs:
-  build:
-    runs-on: self-hosted
+  build_amd64:
+    runs-on: [self-hosted, x64]
     steps:
       - name: Set tag name
         run: |
@@ -29,21 +29,22 @@ jobs:
       - name: Build image
         run: |
           source ./register-sdk-on-host.sh
-          wkdev-sdk-bakery --mode=build --verbose
-          wkdev-sdk-bakery --mode=export --verbose
+          wkdev-sdk-bakery --mode=build --verbose --arch amd64
+          podman image list
+          wkdev-sdk-bakery --mode=export --verbose --arch amd64
 
       - name: Archive image
         uses: actions/upload-artifact@v4
         with:
-          name: wkdev-sdk.tar
-          path: wkdev-sdk.tar
+          name: wkdev-sdk-amd64.tar
+          path: wkdev-sdk-amd64.tar
           retention-days: 7
 
       - name: Test image
         run: |
           CONTAINER="wkdev-$(date +%s)"
           source ./register-sdk-on-host.sh
-          wkdev-create --create-home --home ${HOME}/${CONTAINER}-home --verbose --attach --no-pull --name ${CONTAINER}
+          wkdev-create --create-home --home ${HOME}/${CONTAINER}-home --verbose --attach --no-pull --name ${CONTAINER} --arch amd64
           wkdev-enter -n ${CONTAINER} --exec -- git clone --depth=1 https://github.com/WebKit/WebKit.git
           wkdev-enter -n ${CONTAINER} --exec -- ./WebKit/Tools/Scripts/build-webkit --wpe --release --generate-project-only
           wkdev-enter -n ${CONTAINER} --exec -- ./WebKit/Tools/Scripts/build-webkit --gtk --release --generate-project-only
@@ -53,7 +54,7 @@ jobs:
 
   deploy:
     runs-on: self-hosted
-    needs: build
+    needs: [build_amd64]
     if: github.ref_name == 'main' || startsWith(github.ref_name, 'tag/')
     steps:
       - name: Set tag name
@@ -62,21 +63,26 @@ jobs:
             echo "WKDEV_SDK_TAG=latest" >> "${GITHUB_ENV}"
           fi
           echo "WKDEV_SDK_CONTAINER_REGISTRY_USER_NAME=$(echo ${GITHUB_REPOSITORY_OWNER} | tr '[:upper:]' '[:lower:]')" >> "${GITHUB_ENV}"
+          echo "REPO=ghcr.io/$(echo ${GITHUB_REPOSITORY_OWNER} | tr '[:upper:]' '[:lower:]')/wkdev-sdk" >> "${GITHUB_ENV}"
 
       - name: Install podman
         run: sudo apt-get update && sudo apt-get -y install podman fuse-overlayfs
 
       - name: Checkout repo
         uses: actions/checkout@v4
 
-      - name: Download image
+      - name: Download images
         uses: actions/download-artifact@v4
         with:
-          name: wkdev-sdk.tar
+          pattern: wkdev-sdk-*
+          merge-multiple: true
+      - run: ls -al
 
       - name: Deploy image
         run: |
-          podman load < wkdev-sdk.tar
+          podman load < ./wkdev-sdk-amd64.tar
+          podman image list
           echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io --username=${GITHUB_REPOSITORY_OWNER} --password-stdin
           source ./register-sdk-on-host.sh
-          wkdev-sdk-bakery --mode=deploy --verbose
+          wkdev-sdk-bakery --mode=deploy --verbose --multiarch
+          podman image list

scripts/host-only/wkdev-create

@@ -441,7 +441,18 @@ build_podman_create_arguments() {
         container_arch="${program_options["arch"]}"
         echo "Overriding container architecture: ${container_arch}"
         arguments+=("--arch=${container_arch}")
+
+        if ! podman image exists "$(get_sdk_qualified_name):${container_tag}"; then
+            echo "Image $(get_sdk_qualified_name):${container_tag} does not exist, trying arch-specific version."
+            container_tag="${container_tag}_${container_arch}"
+        fi
+    fi
+
+    if ! podman image exists "$(get_sdk_qualified_name):${container_tag}"; then
+        echo "Image $(get_sdk_qualified_name):${container_tag} does not exist."
+        exit 1
     fi
+    echo "Using image $(get_sdk_qualified_name):${container_tag}."
 
     set +o nounset
     try_process_user ${1}

scripts/host-only/wkdev-sdk-bakery

@@ -22,7 +22,8 @@ argsparse_use_option =env:        "Environment variable as string array, e.g. -e
 argsparse_use_option =mode:       "Operation mode: 'build', 'deploy', or 'export'" mandatory
 argsparse_use_option idle-cores:  "Number of CPU cores to leave idle, when building the image" type:uint default:2
 argsparse_use_option =tag:        "Tag to use for created image." default:$(get_default_container_tag)
-argsparse_use_option =arch:       "Container architecture."
+argsparse_use_option =arch:       "Container architecture. When building images, we also append this arch to the tag name."
+argsparse_use_option multiarch    "Assemble all available images of the form tag_ARCH into one multiarch image."
 
 argsparse_usage_description="$(cat <<EOF
 << Purpose >>
@@ -54,21 +55,25 @@ get_number_of_cores_for_build() { nproc --ignore=${idle_cores}; }
 
 build_image() {
 
+    tag="$(get_tag_for_build)"
+
+    if argsparse_is_option_set "arch"; then
+        container_arch="${program_options["arch"]}"
+        echo "Overriding container architecture: ${container_arch}"
+        podman_argument+=("--arch=${container_arch}")
+
+        tag="${tag}_${container_arch}"
+    fi
+
     _log_ ""
-    _log_ "-> Building container image '${container_image_name}' using tag '$(get_tag_for_build)'..."
+    _log_ "-> Building container image '${container_image_name}' using tag '${tag}'..."
     _log_ ""
 
     pushd "$(get_image_directory_by_name "${container_image_name}")" &>/dev/null
     timer_start
 
     local podman_argument=("--jobs" "$(get_number_of_cores_for_build)")
-    podman_argument+=("--tag" "$(get_tag_for_build)")
-
-    if argsparse_is_option_set "arch"; then
-        container_arch="${program_options["arch"]}"
-        echo "Overriding container architecture: ${container_arch}"
-        podman_argument+=("--arch=${container_arch}")
-    fi
+    podman_argument+=("--tag" "${tag}")
 
     for environment_variable in "${cumulated_values_env[@]}"
     do
@@ -88,6 +93,22 @@ deploy_image() {
 
     local image_directory="$(get_image_directory_by_name "${container_image_name}")"
     pushd "${image_directory}" &>/dev/null || _abort_ "Switching to directory '${image_directory}' failed"
+
+    if argsparse_is_option_set "multiarch"; then
+        image=$(get_qualified_name "${container_image_name}")
+        target_tag="${container_tag}"
+        image_qualified="$(get_tag_for_build)"
+        echo "Building multiarch image for ${image_qualified}"
+        input_tags="$(podman image list "${image}" --format "{{.Tag}}" | grep "${target_tag}_")"
+        run_podman_silent_unless_verbose rmi --ignore "${image_qualified}" || _abort_ "Deleting existing manifest failed"
+        run_podman_silent_unless_verbose manifest create "${image_qualified}" || _abort_ "Creating manifest failed"
+        for input_tag in "${input_tags[@]}"
+        do
+            echo "Adding ${image}:${input_tag} to ${image_qualified}"
+            run_podman_silent_unless_verbose manifest add "${image_qualified}" "containers-storage:${image}:${input_tag}" || _abort_ "Adding to manifest failed"
+        done
+    fi
+
     run_podman_silent_unless_verbose push "$(get_tag_for_build)" || _abort_ "Pushing to registry failed"
     popd &>/dev/null
 }
@@ -98,7 +119,15 @@ export_image() {
     _log_ "-> Exporting container image '${container_image_name}' to file ${container_image_name}.tar..."
     _log_ ""
 
-    run_podman_silent_unless_verbose save --format=oci-archive -o "${container_image_name}.tar" "$(get_tag_for_build)"
+    if argsparse_is_option_set "arch"; then
+        output_file="${container_image_name}-${program_options["arch"]}.tar"
+        tag="$(get_tag_for_build)_${program_options["arch"]}"
+    else
+        output_file="${container_image_name}.tar"
+        tag="$(get_tag_for_build)"
+    fi
+
+    run_podman_silent_unless_verbose save --format=oci-archive -o ${output_file} ${tag}
 }
 
 # Main functionality