Merge pull request #1847 from flatcar/kai/unify-qemu

Set up symlinks for same image artifacts to remove qemu/qemu_uefi_secure

Author: pothos

.github/workflows/ci.yaml

@@ -218,13 +218,23 @@ jobs:
           formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
 
           for format in ${formats}; do
+              if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then
+                  continue
+              fi
               echo " ###################  VENDOR '${format}' ################### "
               ./run_sdk_container -n "${container_name}" \
                   ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                       --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
                       --image_compression_formats=bz2
           done
 
+          # Zip doesn't handle symlinks well, remove them
+          rm -f artifacts/${arch}-usr/latest/flatcar_production_{qemu,qemu_uefi_secure}_image.img*
+          # or create an explicit copy:
+          if [ -e artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz ]; then
+            rm -f artifacts/${arch}-usr/latest/flatcar_production_pxe.vmlinuz
+            cp artifacts/${arch}-usr/latest/flatcar_production_{image,pxe}.vmlinuz
+          fi
           # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
           #  so we move things around.
           mkdir -p artifacts/images

build_library/release_util.sh

@@ -67,9 +67,21 @@ compress_file() {
         ;;
     esac
 
-    ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+    # Check if symlink in which case we set up a "compressed" symlink
+    local compressed_name="${filepath}.${compression_format}"
+    if [ -L "${filepath}" ]; then
+        # We could also test if the target exists and otherwise do the compression
+        # but we might then end up with two different compressed artifacts
+        local link_target
+        link_target=$(readlink -f "${filepath}")
+        local target_basename
+        target_basename=$(basename "${link_target}")
+        ln -fs "${target_basename}.${compression_format}" "${compressed_name}"
+    else
+        ${IMAGE_ZIPPER} -f "${filepath}" 2>&1 >/dev/null || die "failed to compress ${filepath}"
+    fi
 
-    echo -n "${filepath}.${compression_format}"
+    echo -n "${compressed_name}"
 }
 
 compress_disk_images() {
@@ -85,7 +97,7 @@ compress_disk_images() {
     # Files that did not match the filter for disk images.
     local -n local_extra_files="$3"
 
-    info "Compressing images"
+    info "Compressing ${#local_files_to_evaluate[@]} images"
     # We want to compress images, but we also want to remove the uncompressed files
     # from the list of uploadable files.
     for filename in "${local_files_to_evaluate[@]}"; do
@@ -98,6 +110,9 @@ compress_disk_images() {
 
             # An associative array we set an element on whenever we process a format.
             # This way we don't process the same format twice. A unique for array elements.
+            # (But first we need to unset the previous loop or we can only compress a single
+            # file per list of files).
+            unset processed_format
             declare -A processed_format
             for format in "${FORMATS[@]}";do
                 if [ -z "${processed_format[${format}]}" ]; then
@@ -113,7 +128,10 @@ compress_disk_images() {
                [ "${filename##*/}" != "flatcar_production_image.bin" ] &&
                [ "${filename##*/}" != "flatcar_production_update.bin" ] &&
                ! echo "${FORMATS[@]}" | grep -q "none"; then
+                info "Removing ${filename}"
                 rm "${filename}"
+            else
+                info "Keeping ${filename}"
             fi
         else
             local_extra_files+=( "${filename}" )            

build_library/vm_image_util.sh

@@ -23,9 +23,7 @@ VALID_IMG_TYPES=(
     packet
     parallels
     pxe
-    qemu
     qemu_uefi
-    qemu_uefi_secure
     rackspace
     rackspace_onmetal
     rackspace_vhd
@@ -126,27 +124,13 @@ IMG_DEFAULT_MEM=2048
 IMG_DEFAULT_CPUS=2
 
 ## qemu
-IMG_qemu_DISK_FORMAT=qcow2
-IMG_qemu_DISK_LAYOUT=vm
-IMG_qemu_CONF_FORMAT=qemu
-IMG_qemu_OEM_USE=qemu
-IMG_qemu_OEM_PACKAGE=common-oem-files
-IMG_qemu_OEM_SYSEXT=oem-qemu
-
 IMG_qemu_uefi_DISK_FORMAT=qcow2
 IMG_qemu_uefi_DISK_LAYOUT=vm
 IMG_qemu_uefi_CONF_FORMAT=qemu_uefi
 IMG_qemu_uefi_OEM_USE=qemu
 IMG_qemu_uefi_OEM_PACKAGE=common-oem-files
 IMG_qemu_uefi_OEM_SYSEXT=oem-qemu
 
-IMG_qemu_uefi_secure_DISK_FORMAT=qcow2
-IMG_qemu_uefi_secure_DISK_LAYOUT=vm
-IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure
-IMG_qemu_uefi_secure_OEM_USE=qemu
-IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files
-IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu
-
 ## xen
 IMG_xen_CONF_FORMAT=xl
 
@@ -326,7 +310,7 @@ get_default_vm_type() {
     local board="$1"
     case "$board" in
     amd64-usr)
-        echo "qemu"
+        echo "qemu_uefi"
         ;;
     arm64-usr)
         echo "qemu_uefi"
@@ -603,6 +587,18 @@ write_vm_disk() {
     info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
 
+    # We now only support building qemu_uefi and set up symlinks
+    # for the qemu and qemu_uefi_secure images
+    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
+        local qemu="${VM_DST_IMG/qemu_uefi/qemu}"
+        local qemu_uefi_secure="${VM_DST_IMG/qemu_uefi/qemu_uefi_secure}"
+        local target_basename
+        target_basename=$(basename "${VM_DST_IMG}")
+        ln -fs "${target_basename}" "${qemu}"
+        ln -fs "${target_basename}" "${qemu_uefi_secure}"
+        VM_GENERATED_FILES+=( "${qemu}" "${qemu_uefi_secure}" )
+    fi
+
     # Add disk image to final file list if it isn't going to be bundled
     if [[ -z "$(_get_vm_opt BUNDLE_FORMAT)" ]]; then
         VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
@@ -693,7 +689,7 @@ _write_cpio_disk() {
     local grub_name="$(_dst_name "_grub.efi")"
     _write_cpio_common $@
     # Pull the kernel and loader out of the filesystem
-    cp "${base_dir}"/boot/flatcar/vmlinuz-a "${dst_dir}/${vmlinuz_name}"
+    ln -fs flatcar_production_image.vmlinuz "${dst_dir}/${vmlinuz_name}"
 
     local grub_arch
     case $BOARD in
@@ -809,6 +805,13 @@ _write_qemu_uefi_conf() {
     sed -e "s%^VM_PFLASH_RO=.*%VM_PFLASH_RO='${flash_ro}'%" \
         -e "s%^VM_PFLASH_RW=.*%VM_PFLASH_RW='${flash_rw}'%" -i "${script}"
     VM_GENERATED_FILES+=( "$(_dst_dir)/${flash_ro}" "$(_dst_dir)/${flash_rw}" )
+
+    # We now only support building qemu_uefi and generate the
+    # other artifacts from here
+    if [ "${VM_IMG_TYPE}" = qemu_uefi ]; then
+      VM_IMG_TYPE=qemu _write_qemu_conf
+      VM_IMG_TYPE=qemu_uefi_secure _write_qemu_uefi_secure_conf
+    fi
 }
 
 _write_qemu_uefi_secure_conf() {

changelog/changes/2024-04-08-unify-qemu-images.md

@@ -0,0 +1 @@
+- SDK: Unified qemu image formats, so that the `qemu_uefi` build target provides the regular `qemu` and the `qemu_uefi_secure` artifacts ([scripts#1847](https://github.com/flatcar/scripts/pull/1847))

ci-automation/vms.sh

@@ -125,6 +125,9 @@ function _vm_build_impl() {
             mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input"
 
     for format in ${formats}; do
+        if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then
+           continue
+        fi
         echo " ###################  VENDOR '${format}' ################### "
         COMPRESSION_FORMAT="bz2"
         if [[ "${format}" =~ ^(openstack_mini|digitalocean)$ ]];then