Merge pull request #1216 from flatcar/contrib/torcx-deprecation-docker-sysext

Contrib: Deprecate torcx, ship containerd / docker as sysexts

Author: t-lo

.github/workflows/ci.yaml

@@ -100,14 +100,8 @@ jobs:
           [ -z "${{ inputs.image_formats }}" ] || IMAGE_FORMATS="${{ inputs.image_formats }}"
           echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV
 
-          # Artifact root for images and torcx tarball as seen from within the container
+          # Artifact root for images as seen from within the container
           echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
-          echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
-          mkdir -p artifacts/torcx
-
-          # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
-          #  this with its IP address.
-          echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
 
           if [ -n "${{ inputs.custom_sdk_version }}" ] ; then
               echo "CUSTOM_SDK_VERSION=${{ inputs.custom_sdk_version }}" >> $GITHUB_ENV
@@ -146,9 +140,7 @@ jobs:
           #  which will be re-used by subsequent build steps.
           ./run_sdk_container -n "${container_name}" -v "${version}" \
             -C "${sdk_image}" \
-            ./build_packages --board="${arch}-usr" \
-                --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
-                --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"
+            ./build_packages --board="${arch}-usr"
 
           # Create binpkgs tarball for archiving as artifact later
           ./run_sdk_container -n "${container_name}" \
@@ -193,7 +185,7 @@ jobs:
           ./run_sdk_container -n "${container_name}" \
                   ./build_image --board="${arch}-usr" --group="${channel}" \
                                 --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
-                                --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container
+                                prodtar container
 
       - name: Generate reports
         shell: bash
@@ -291,12 +283,6 @@ jobs:
             mv * ../../images/
           )
 
-          # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
-          #  (such as "docker:20.10.torcx.tgz")
-          mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
-          tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .
-
-
       - name: Upload binpkgs
         uses: actions/upload-artifact@v3
         with:
@@ -336,14 +322,6 @@ jobs:
           path: |
             scripts/artifacts/images/flatcar_developer_container*
 
-      - name: Upload torcx tarball
-        uses: actions/upload-artifact@v3
-        with:
-          retention-days: 7
-          name: ${{ matrix.arch }}-torcx
-          path: |
-            scripts/torcx.tar
-
       - name: Upload reports
         uses: actions/upload-artifact@v3
         with:

.github/workflows/containerd-apply-patch.sh

@@ -28,21 +28,13 @@ git mv "${containerdEbuildOldSymlink}" "${containerdEbuildNewSymlink}"
 sed -i "s/CONTAINERD_COMMIT=\"\(.*\)\"/CONTAINERD_COMMIT=\"${COMMIT_HASH}\"/g" "${containerdEbuildMain}"
 sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${containerdEbuildMain}"
 
-
-DOCKER_VERSION=$(sed -n "s/^DIST docker-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${DOCKER_VERSION%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/containerd-${VERSION_OLD}/containerd-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 popd
 
 URL="https://github.com/containerd/containerd/releases/tag/v${VERSION_NEW}"
 
 generate_update_changelog 'containerd' "${VERSION_NEW}" "${URL}" 'containerd'
 
-commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}" \
-               app-torcx/docker
+commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}"
 
 cleanup_repo
 

.github/workflows/docker-apply-patch.sh

@@ -33,12 +33,6 @@ git mv "${cliEbuildOld}" "${cliEbuildNew}"
 sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_CLI}/g" "${cliEbuildNew}"
 sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${cliEbuildNew}"
 
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${VERSION_OLD%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/docker-${VERSION_OLD}/docker-${VERSION_NEW}/g" "${torcxEbuildFile}"
-sed -i "s/docker-cli-${VERSION_OLD}/docker-cli-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 # update also docker versions used by the current runc ebuild file.
 versionRunc=$(sed -n "s/^DIST runc-\([0-9]*.[0-9]*.*\)\.tar.*/\1/p" app-containers/runc/Manifest | sort -ruV | head -n1)
 runcEbuildFile=$(get_ebuild_filename app-containers/runc "${versionRunc}")
@@ -63,7 +57,6 @@ generate_update_changelog 'Docker' "${VERSION_NEW}" "${URL}" 'docker'
 regenerate_manifest app-containers/docker-cli "${VERSION_NEW}"
 commit_changes app-containers/docker "${VERSION_OLD}" "${VERSION_NEW}" \
                app-containers/docker-cli \
-               app-torcx/docker \
                app-containers/runc
 
 cleanup_repo

.github/workflows/run-kola-tests.yaml

@@ -99,12 +99,6 @@ jobs:
         with:
           name: ${{ matrix.arch }}-devcontainer
 
-      - name: Download torcx tarball
-        if: ${{ !inputs.workflow_run_id }}
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ matrix.arch }}-torcx
-
       - name: Download binpkgs from other workflow
         uses: gabriel-samfira/action-download-artifact@v5
         if: ${{ inputs.workflow_run_id }}
@@ -141,24 +135,15 @@ jobs:
           run_id: ${{ inputs.workflow_run_id }}
           name: ${{ matrix.arch }}-devcontainer
 
-      - name: Download torcx tarball from other workflow
-        uses: gabriel-samfira/action-download-artifact@v5
-        if: ${{ inputs.workflow_run_id }}
-        with:
-          workflow: ${{ inputs.workflow_name_or_id }}
-          workflow_conclusion: success
-          run_id: ${{ inputs.workflow_run_id }}
-          name: ${{ matrix.arch }}-torcx
-
       - name: Extract artifacts
         shell: bash
         run: |
           exec 2>&1
           set -x
           set -euo pipefail
 
-          # Set up a webserver for devcontainer and torcx tests.
-          # The respective tests will download devcontainer and torcx tarball via http.
+          # Set up a webserver for devcontainer tests.
+          # The respective tests will download devcontainer via http.
           # The devcontainer test will then run a build
           #   which will download and install binpkgs into the dev container.
           # For the sake of that test we will serve both via a temporary local web server.
@@ -174,19 +159,6 @@ jobs:
           mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT}
           tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar
 
-          tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar
-
-          # Move torcx package into plain webroot
-          #  (path consists of <arch>/<packagename>/<checksum>/<packagename>:<version>.torcx.tar.gz)
-          mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \
-             "${TESTS_WEBSERVER_WEBROOT}"
-
-          # Update torcx.json's http URL to point to the webserver IP.
-          # ci.yaml defines the "localhost" placeholder in its "Set Environment" step.
-          sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \
-                 "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
-          cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json"
-
           # Extract the generic image we'll use for qemu tests.
           # Note that the qemu[_uefi] tests use the generic image instead of the
           #  qemu vendor VM image ("Astronaut: [...] Always have been.").
@@ -221,14 +193,6 @@ jobs:
 
           source ci-automation/test.sh
 
-          # Provide our own torcx prepare function so we use our local manifest json.
-          # This is called by test_run below.
-          function __prepare_torcx() {
-            shift; shift # no need for arch or vernum
-            local destdir="$1"
-            cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}"
-          }
-
           PARALLEL_ARCH=10
 
           cat > sdk_container/.env <<EOF

.github/workflows/runc-apply-patch.sh

@@ -36,20 +36,14 @@ sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" app-containers/containerd/c
 
 dockerVersion=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
 
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${dockerVersion%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 popd
 
 URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}"
 
 generate_update_changelog 'runc' "${VERSION_NEW}" "${URL}" 'runc'
 
 commit_changes app-containers/runc "${VERSION_OLD}" "${VERSION_NEW}" \
-               app-containers/containerd \
-               app-torcx/docker
+               app-containers/containerd
 
 cleanup_repo
 

build_image

@@ -33,10 +33,8 @@ DEFINE_string base_pkg "coreos-base/coreos" \
   "The base portage package to base the build off of (only applies to prod images)"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
   "The base portage package to base the build off of (only applies to dev containers)"
-DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \
-  "The torcx manifest describing torcx packages for this image (or blank for none)"
-DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \
-  "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set."
+DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker" \
+  "Comma-separated list of name:package - build 'package' into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \
@@ -91,11 +89,6 @@ switch_to_strict_mode
 
 check_gsutil_opts
 
-# Patch around default values not being able to depend on other flags.
-if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then
-  FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json"
-fi
-
 # If downloading packages is enabled ensure the board is configured properly.
 if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then
   "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \
@@ -110,7 +103,6 @@ fi
 . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/dev_container_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
-. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1
 
 PROD_IMAGE=0
@@ -175,7 +167,7 @@ fi
 
 if [[ "${PROD_IMAGE}" -eq 1 ]]; then
   IMAGE_BUILD_TYPE="prod"
-  create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg}
+  create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg} ${FLAGS_base_sysexts}
   if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]]; then
     generate_update "${FLATCAR_PRODUCTION_IMAGE_NAME}" ${DISK_LAYOUT}
   elif [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then

build_library/build_image_util.sh

@@ -260,8 +260,8 @@ image_packages_portage() {
     ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \
         equery --no-color list --format '$cpv::$repo' '*'
 }
-# List packages implicitly contained in rootfs, such as in torcx packages or
-# initramfs.
+
+# List packages implicitly contained in rootfs, such as in initramfs.
 image_packages_implicit() {
     local profile="${BUILD_DIR}/configroot/etc/portage/profile"
 
@@ -290,11 +290,6 @@ image_packages_implicit() {
             query_available_package "${pkg}"
         done < "${profile}/package.provided"
     fi
-
-    # Include source packages of all torcx images installed on disk.
-    [ -z "${FLAGS_torcx_manifest}" ] ||
-    torcx_manifest::sources_on_disk "${FLAGS_torcx_manifest}" |
-    while read pkg ; do query_available_package "${pkg}" ; done
 }
 
 # Generate a list of packages installed in an image.
@@ -517,8 +512,6 @@ EOF
 # Add /usr/share/SLSA reports for packages indirectly contained within the rootfs
 # If the package is available in BOARD_ROOT accesses it from there, otherwise
 # needs to download binpkg.
-# Reports for torcx packages are also included when adding the torcx package to
-# rootfs.
 insert_extra_slsa() {
   info "Inserting additional SLSA file"
   local rootfs="$1"
@@ -626,27 +619,6 @@ finish_image() {
   local install_grub=0
   local disk_img="${BUILD_DIR}/${image_name}"
 
-  # Copy in packages from the torcx store that are marked as being on disk
-  if [ -n "${FLAGS_torcx_manifest}" ]; then
-    for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do
-      local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")"
-      for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do
-        local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
-        if [[ -n "${on_disk_path}" ]]; then
-          local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")"
-          sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \
-            "${root_fs_dir}${on_disk_path}"
-          sudo tar xf "${root_fs_dir}${on_disk_path}" -C "${root_fs_dir}" --wildcards "./usr/share/SLSA"
-          if [[ "${version}" == "${default_version}" ]]; then
-            # Create the default symlink for this package
-            sudo ln -fns "${on_disk_path##*/}" \
-              "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz"
-          fi
-        fi
-      done
-    done
-  fi
-
   # Only enable rootfs verification on prod builds.
   local disable_read_write="${FLAGS_FALSE}"
   if [[ "${IMAGE_BUILD_TYPE}" == "prod" ]]; then