X-Smart-Squash: Squashed 50 commits:

0116bf7 ccache
7fe4a87 typo
4ba08ca ansible format
31fb575 add .ccache to .gitignore
f5e5fd3 ccache on builder image
4c75f3a debug logs for builder
3412c94 create logs
29778a0 typo
b22b325 print
3ba1265 key
a414bb1 remove debug
d12f47b enable s390x caching
5578333 typo
10b4a40 s390x
9c4ffa1 skip docker inject action on s390x
66d4e7f fix path
4a58a89 builtin
f16c110 fix
deda8ba format
23d5ed4 var
ef8e17a order
65b1816 fmt
2378492 simplify
89c69c6 scripts
4170881 version
1b191f8 path
be6e04b task time
b8f61ba quote
30382e5 cache file for remote vm
b8c982b fmt
15c70af var
a2f148b update
a927675 check dir
a77bd93 enable builder, s390x collector
0ba7d0a typo
664e31a dbg
28ace91 broken cache
f822219 ls
9d5ccdf rm
8ce9e57 slash
e80f8f5 slash
1482ea1 permissions and fix builder ccache
3fb4a1a s390x
4ee116e s390x
0887d1d root
6bb2700 dbg
b9e8ed1 Revert "dbg"
ab60dff rm opts
c85e21e collector
bdb810b no ccache avail

Author: robbycochran

.github/workflows/collector-builder.yml

@@ -74,6 +74,27 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up builder ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/builder/.ccache
+          key: builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            builder-ccache-${{ matrix.arch }}-master-
+
+      - name: Setup builder ccache in docker cache
+        if: |
+          (github.event_name != 'pull_request' && matrix.arch != 's390x') ||
+          matrix.arch == 'amd64' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+        with:
+          cache-map: |
+            {
+              "${{ github.workspace }}/builder/.ccache": "/root/.ccache"
+            }
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
@@ -152,18 +173,30 @@ jobs:
           (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
         timeout-minutes: 480
         run: |
+          [ ! -d "${{ github.workspace }}/builder/.ccache" ] && rm -f "${{ github.workspace }}/builder/.ccache"
+          ls -al "${{ github.workspace }}/builder/.ccache" || true
           ansible-playbook \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
+          ls -al "${{ github.workspace }}/builder/.ccache" || true
 
       - name: Destroy VMs
         if: always() && matrix.arch == 's390x'
         run: |
           make -C ansible destroy-vms
 
+      - name: Store artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: builder-${{ matrix.arch }}-logs
+          path: |
+            ${{ github.workspace }}/build_builder_image.log
+
   create-multiarch-manifest:
     needs:
     - build-builder-image

.github/workflows/collector.yml

@@ -45,6 +45,14 @@ jobs:
         with:
           submodules: true
 
+      - name: Set up ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            ccache-${{ matrix.arch }}-master-
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -84,6 +92,10 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 
+      - name: Stats for ccache after build
+        run: |
+          ccache -s "${{ github.workspace }}/.ccache"
+
   build-collector-image-remote-vm:
     name: Build Collector on a remote VM
     runs-on: ubuntu-latest
@@ -123,6 +135,15 @@ jobs:
           vm-type: rhel-${{ matrix.arch }}
           job-tag: builder
 
+      - name: Set up ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            ccache-${{ matrix.arch }}-master-
+
       - name: Create Build VMs
         run: |
           make -C "${{ github.workspace }}/ansible" create-build-vms
@@ -152,6 +173,7 @@ jobs:
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 

.gitignore

@@ -1,5 +1,7 @@
 .idea/
 .rox/
+.ccache/
+builder/.ccache/
 
 integration-tests/container-logs/
 integration-tests/*.log

Makefile

@@ -6,6 +6,8 @@ NPROCS ?= $(shell nproc)
 DEV_SSH_SERVER_KEY ?= $(CURDIR)/.collector_dev_ssh_host_ed25519_key
 BUILD_BUILDER_IMAGE ?= false
 
+CCACHE_DIR?=$(CURDIR)/.ccache
+
 export COLLECTOR_VERSION := $(COLLECTOR_TAG)
 
 .PHONY: tag
@@ -25,9 +27,11 @@ container-dockerfile-dev:
 builder:
 ifneq ($(BUILD_BUILDER_IMAGE), false)
 	docker buildx build --load --platform ${PLATFORM} \
+		--build-arg USE_CCACHE="${USE_CCACHE}" \
 		-t quay.io/stackrox-io/collector-builder:$(COLLECTOR_BUILDER_TAG) \
 		-f "$(CURDIR)/builder/Dockerfile" \
-		"$(CURDIR)/builder"
+		"$(CURDIR)/builder" \
+		2>&1 | tee build_builder_image.log
 endif
 
 collector: check-builder
@@ -84,6 +88,7 @@ start-builder: builder teardown-builder
 		--name $(COLLECTOR_BUILDER_NAME) \
 		--pull missing \
 		--platform ${PLATFORM} \
+		-v $(CCACHE_DIR):/root/.ccache \
 		-v $(CURDIR):$(CURDIR) \
 		$(if $(LOCAL_SSH_PORT),-p $(LOCAL_SSH_PORT):22 )\
 		-w $(CURDIR) \

Makefile-constants.mk

@@ -19,3 +19,4 @@ BPF_DEBUG_MODE ?= false
 
 COLLECTOR_BUILD_CONTEXT = collector/
 COLLECTOR_BUILDER_NAME ?= collector_builder_$(HOST_ARCH)
+USE_CCACHE ?= true

ansible/ansible.cfg

@@ -5,6 +5,7 @@ display_skipped_hosts = false
 host_key_checking = false
 remote_tmp = /tmp/ansible
 forks = 20
+callbacks_enabled = profile_tasks
 
 [ssh_connection]
 ssh_args = -o StrictHostKeyChecking=no -C -o ControlMaster=auto -o ControlPersist=60s -o ServerAliveInterval=30 -o ServerAliveCountMax=10

ansible/ci-build-builder.yml

@@ -1,5 +1,5 @@
 ---
-- name: Build and push collector image
+- name: Build and push builder image
   hosts: "{{ build_hosts | default('all') }}"
 
   environment:
@@ -10,6 +10,10 @@
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: "builder/.ccache"
+    ccache_archive: "docker.tar.gz"
+    ccache_path: "{{ ccache_dir }}/{{ ccache_archive }}"
+    container_ccache_dir: "/root/.ccache"
 
   tasks:
     - name: Clone repository
@@ -21,13 +25,55 @@
         # than with commit hashes, prevents "reference is not a tree" errors
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
+        depth: 1
         recursive: true
       when: arch == "s390x"
 
+    - name: Check if ccache archive exists
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_path }}"
+      register: ccache_check
+      when: arch == "s390x"
+
+    - name: Copy ccache to build
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_path }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}/"
+      when: arch == "s390x" and ccache_check.stat.exists
+
+    - name: Inject docker cache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/inject_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x" and ccache_check.stat.exists
+
     - name: Build the collector builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: builder
+        params:
+          USE_CCACHE: "true"
+
+    - name: Extract docker cache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/extract_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Fetch ccache
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_path }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: arch == "s390x"
+
+    - name: Fetch build log
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/build_builder_image.log"
+        dest: "{{ github_workspace }}/"
+        flat: yes
+      when: arch == "s390x"
 
     - name: Retag collector builder image to arch specific
       community.docker.docker_image:

ansible/ci-build-collector.yml

@@ -12,6 +12,8 @@
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: ".ccache"
+    ccache_archive: "docker.tar.gz"
 
   tasks:
     - debug: var=collector_root
@@ -25,8 +27,29 @@
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
         recursive: true
+        depth: 1
       when: arch == "s390x"
 
+    - name: Check if remote ccache archive exists on controller
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+      register: ccache_check
+      when: arch == "s390x"
+
+    - name: Copy ccache from the controller to build VM
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/"
+      when: arch == "s390x" and ccache_check.stat.exists
+
+    - name: Extract ccache archive if found
+      ansible.builtin.unarchive:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}"
+        remote_src: true
+      when: arch == "s390x" and ccache_check.stat.exists
+
     - name: Run the builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
@@ -36,10 +59,27 @@
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: image
+        params:
+          CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
+          USE_CCACHE: true
       register: build_result
       # ensure this action is printed
       tags: [print_action]
 
+    - name: Create ccache archive
+      ansible.builtin.shell: |
+        rm -f "{{ collector_root }}/{{ ccache_archive }}"
+        cd "{{ collector_root }}/{{ ccache_dir }}"
+        tar czf "{{ collector_root }}/{{ ccache_archive }}" .
+      when: arch == "s390x"
+
+    - name: Copy ccache from build machine to controller
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: arch == "s390x"
+
     - name: Retag collector image to arch specific
       community.docker.docker_image:
         name: "{{ collector_image }}"

ansible/scripts/extract_docker_cache.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# based on https://github.com/reproducible-containers/buildkit-cache-dance
+
+set -e
+
+usage() {
+    echo "Usage: $0 <cache_destination_directory> <target_path_inside_container>"
+    exit 1
+}
+
+if [ "$#" -ne 2 ]; then
+    usage
+fi
+
+CACHE_DEST="$1"
+TARGET_PATH="$2"
+
+CCACHE_ARCHIVE="docker.tar.gz"
+
+SCRATCH_DIR=$(mktemp -d)
+SCRATCH_ARCHIVE=$(mktemp)
+# shellcheck disable=SC2064
+trap "rm -rf ${SCRATCH_DIR}; rm -f ${SCRATCH_ARCHIVE}" EXIT
+
+# Timestamp to bust cache
+date -Iseconds > "${SCRATCH_DIR}/buildstamp"
+
+cat << EOF > "${SCRATCH_DIR}/Dockerfile.extract"
+FROM busybox:1
+COPY buildstamp buildstamp
+RUN --mount=type=cache,target=${TARGET_PATH} \\
+    mkdir -p /var/docker-cache/ \\
+    && cp -p -R ${TARGET_PATH}/. /var/docker-cache/ || true
+EOF
+
+echo "Generated Dockerfile.extract"
+cat "${SCRATCH_DIR}/Dockerfile.extract"
+
+# Build the image and load it into Docker
+docker buildx build -f "${SCRATCH_DIR}/Dockerfile.extract" --tag cache:extract --load "${SCRATCH_DIR}"
+docker images
+
+# Remove any existing cache-container
+docker rm -f cache-container || true
+
+# Create a container from cache:extract
+docker create --name cache-container cache:extract
+docker ps
+
+# Extract the cache from the container
+docker cp -L cache-container:/var/docker-cache - | tar -H posix -x -C "${SCRATCH_DIR}"
+ls "${SCRATCH_DIR}"
+
+# Compress the cache from the container
+(cd "${SCRATCH_DIR}/docker-cache" && chmod -R 777 . && tar czf "${SCRATCH_ARCHIVE}" .)
+
+# Move the cache into its dest
+rm -f "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+mv "${SCRATCH_ARCHIVE}" "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+chmod 666 "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+
+echo "Docker cache extraction completed successfully."