Backport P, Z and Arm runners to 3.21 (#2254)

* ROX-29116: (fix) Use ARM GH action workflow runners for ARM builds (#2106)

* Revert "Revert GH arm changes (#2107)"
* Add arm64  stable, beta, and dev channels for Google COS integration tests (#2096)

* Use P and Z GHA runners (#2215)

* Use go cross-compilation to speed up test builds on CI (#2210)

After adding Arm runners on GHA, we made it so the Arm test image is
always built. Unfortunately, because we use QEMU to build the images,
the build has gotten quite slow.

In order to speed up the build, this change makes it so we cross-compile
the binaries for all our supported platforms locally and the image build
simply copies the binaries into the final image.

---------

Co-authored-by: Robby Cochran <rcochran@redhat.com>

Author: Molter73

.github/workflows/collector-builder.yml

@@ -8,10 +8,11 @@ on:
         required: true
         description: |
           The tag used to build the collector image
+
     outputs:
       collector-builder-tag:
         description: The builder tag used by the build
-        value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }}
+        value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag }}
 
 env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
@@ -23,7 +24,11 @@ jobs:
     name: Determine if builder image needs to be built
     runs-on: ubuntu-24.04
     outputs:
-      build-image: ${{ steps.changed.outputs.builder-changed }}
+      build-image: ${{ steps.builder-tag.outputs.build-image || false }}
+      collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}}
+
+    env:
+      DEFAULT_BUILDER_TAG: master
 
     steps:
       - uses: actions/checkout@v4
@@ -38,106 +43,76 @@ jobs:
               - builder/Dockerfile
               - .github/workflows/collector-builder.yml
 
+      - name: Check labels and define builder tag
+        id: builder-tag
+        if: |
+          steps.changed.outputs.builder-changed == 'true' ||
+          (github.event_name == 'push' && (
+            github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
+          )) ||
+          contains(github.event.pull_request.labels.*.name, 'build-builder-image') ||
+          github.event_name == 'schedule'
+        run: |
+          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
+          if [[ "${{ github.event_name }}" == 'pull_request' || \
+                "${{ github.ref_type }}" == 'tag' || \
+                "${{ github.ref_name }}" =~ ^release- ]]; then
+            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
+          fi
+
+          echo "::notice::Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}"
+          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
+          echo "build-image=true" >> "$GITHUB_OUTPUT"
+
   build-builder-image:
-    name: Build the builder image
-    runs-on: ubuntu-24.04
-    # Multiarch builds sometimes take for eeeeeeeeeever
-    timeout-minutes: 480
+    name: Build builder image
     needs:
     - builder-needs-rebuilding
     if: |
-      needs.builder-needs-rebuilding.outputs.build-image == 'true' ||
-      (github.event_name == 'push' && (
-        github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
-      )) ||
-      contains(github.event.pull_request.labels.*.name, 'build-builder-image') ||
-      github.event_name == 'schedule'
-    outputs:
-      collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }}
+      needs.builder-needs-rebuilding.outputs.build-image == 'true'
     strategy:
-      fail-fast: false
       matrix:
-        arch: [amd64, ppc64le, s390x, arm64]
+        arch:
+        - amd64
+        - arm64
+        - ppc64le
+        - s390x
+    runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') ||
+                 (matrix.arch == 'ppc64le' && 'ubuntu-24.04-ppc64le') ||
+                 (matrix.arch == 's390x' && 'ubuntu-24.04-s390x') ||
+                'ubuntu-24.04' }}
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
       BUILD_TYPE: ci
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
 
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          image: tonistiigi/binfmt:qemu-v8.1.5
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - uses: 'google-github-actions/auth@v2'
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
-
-      - uses: 'google-github-actions/setup-gcloud@v2'
-
-      - uses: ./.github/actions/setup-vm-creds
-        with:
-          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
-          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
-          s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
-          ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}
-          s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }}
-          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
-          redhat-username: ${{ secrets.REDHAT_USERNAME }}
-          redhat-password: ${{ secrets.REDHAT_PASSWORD }}
-          vm-type: all
-          job-tag: builder
-
-      - name: Create Build VMs
-        if: |
-          matrix.arch == 's390x' &&
-          (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
+      - name: Install ansible on P&Z runners
+        if: matrix.arch == 'ppc64le' || matrix.arch == 's390x'
         run: |
-          make -C "${{ github.workspace }}/ansible" create-build-vms
-
-      - name: Define builder tag
-        id: builder-tag
-        run: |
-          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
-          if [[ "${{ github.event_name }}" == 'pull_request' || \
-                "${{ github.ref_type }}" == 'tag' || \
-                "${{ github.ref_name }}" =~ ^release- ]]; then
-            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
-          fi
-
-          echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV"
-          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
+          sudo apt-get install -y ansible
 
       - name: Create ansible vars
         run: |
-          {
-            echo "---"
-            echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}"
-            echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}"
-            echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}"
-            echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
-            echo "collector_git_ref: ${{ github.ref }}"
-            echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
-          } > ${{ github.workspace }}/ansible/secrets.yml
+          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
+          ---
+          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
+          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
+          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+          collector_git_ref: ${{ github.ref }}
+          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
+          EOF
 
       - name: Build images
-        if: |
-          (github.event_name != 'pull_request' && matrix.arch != 's390x') ||
-          matrix.arch == 'amd64' ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
-        timeout-minutes: 480
         run: |
           ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
@@ -148,36 +123,18 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
 
-      - name: Build s390x images
-        if: |
-          (github.event_name != 'pull_request' && matrix.arch == 's390x') ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
-        timeout-minutes: 480
-        run: |
-          ansible-playbook \
-            -i ansible/ci \
-            -e build_hosts='job_id_${{ env.JOB_ID }}' \
-            -e arch='${{ matrix.arch }}' \
-            -e @'${{ github.workspace }}/ansible/secrets.yml' \
-            ansible/ci-build-builder.yml
-
-      - name: Destroy VMs
-        if: always() && matrix.arch == 's390x'
-        run: |
-          make -C ansible destroy-vms
-
   create-multiarch-manifest:
     needs:
+    - builder-needs-rebuilding
     - build-builder-image
     name: Create Multiarch manifest
     runs-on: ubuntu-24.04
     if: |
-      github.event_name != 'pull_request' ||
-      (needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
-       contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
+      always() && !contains(join(needs.*.result, ','), 'failure') &&
+      needs.builder-needs-rebuilding.outputs.build-image == 'true'
     env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
-      ARCHS: amd64 ppc64le s390x arm64
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
+      ARCHS: amd64 arm64 ppc64le s390x
 
     steps:
       - uses: actions/checkout@v4
@@ -208,45 +165,12 @@ jobs:
           base-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}
           archs: ${{ env.ARCHS }}
 
-  retag-x86-image:
-    needs:
-    - build-builder-image
-    name: Retag x86 builder image
-    runs-on: ubuntu-24.04
-    if: |
-      github.event_name == 'pull_request' &&
-      needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
-      !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
-    env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
-    steps:
-      - name: Pull image to retag
-        run: |
-          docker pull "quay.io/stackrox-io/collector-builder:${COLLECTOR_BUILDER_TAG}-amd64"
-
-      - name: Retag and push stackrox-io
-        uses: stackrox/actions/images/retag-and-push@v1
-        with:
-          src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64
-          dst-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}
-          username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
-          password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
-
-      - name: Retag and push rhacs-eng
-        uses: stackrox/actions/images/retag-and-push@v1
-        with:
-          src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64
-          dst-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}
-          username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
-          password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
-
   notify:
     runs-on: ubuntu-24.04
     if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request'
     needs:
       - build-builder-image
       - create-multiarch-manifest
-      - retag-x86-image
     steps:
       - name: Slack notification
         uses: rtCamp/action-slack-notify@v2