From d441fd3b03fe2e97e1a9521047b93f7f2da1ab15 Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Fri, 27 Sep 2024 16:06:37 -0700
Subject: [PATCH 01/12] X-Smart-Branch-Parent: origin/master


From 82d352fcc24c7545662dae05a32e78942c20ddcb Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sat, 14 Sep 2024 10:29:43 -0700
Subject: [PATCH 02/12] X-Smart-Squash: Squashed 50 commits:

0116bf706 ccache
7fe4a875b typo
4ba08ca6e ansible format
31fb5751a add .ccache to .gitignore
f5e5fd3b4 ccache on builder image
4c75f3a4e debug logs for builder
3412c949d create logs
29778a0a8 typo
b22b32554 print
3ba126569 key
a414bb180 remove debug
d12f47b1c enable s390x caching
557833344 typo
10b4a406e s390x
9c4ffa14c skip docker inject action on s390x
66d4e7faf fix path
4a58a89df builtin
f16c110d8 fix
deda8ba9c format
23d5ed49f var
ef8e17a00 order
65b1816da fmt
23784923f simplify
89c69c6f7 scripts
4170881fd version
1b191f8b7 path
be6e04b01 task time
b8f61ba23 quote
30382e540 cache file for remote vm
b8c982bf1 fmt
15c70af7c var
a2f148b40 update
a92767545 check dir
a77bd9397 enable builder, s390x collector
0ba7d0ad7 typo
664e31a77 dbg
28ace912f broken cache
f822219c5 ls
9d5ccdfee rm
8ce9e5767 slash
e80f8f530 slash
1482ea122 permissions and fix builder ccache
3fb4a1a06 s390x
4ee116edf s390x
0887d1d06 root
6bb2700ef dbg
b9e8ed1da Revert "dbg"
ab60dff26 rm opts
c85e21edc collector
bdb810b96 no ccache avail
---
 .github/workflows/collector-builder.yml | 33 +++++++++++++
 .github/workflows/collector.yml         | 22 +++++++++
 .gitignore                              |  2 +
 Makefile                                |  7 ++-
 Makefile-constants.mk                   |  1 +
 ansible/ansible.cfg                     |  1 +
 ansible/ci-build-builder.yml            | 48 ++++++++++++++++++-
 ansible/ci-build-collector.yml          | 40 ++++++++++++++++
 ansible/scripts/extract_docker_cache.sh | 62 +++++++++++++++++++++++++
 ansible/scripts/inject_docker_cache.sh  | 47 +++++++++++++++++++
 builder/Dockerfile                      |  6 ++-
 builder/install/10-gperftools.sh        |  5 ++
 builder/install/50-jq.sh                |  5 ++
 builder/install/50-libb64.sh            |  5 ++
 builder/install/60-tbb.sh               |  5 ++
 builder/install/80-libbpf.sh            |  5 ++
 builder/install/ccache.sh               | 23 +++++++++
 builder/install/install-dependencies.sh | 31 ++++++++++++-
 collector/CMakeLists.txt                |  8 ++++
 collector/Makefile                      |  1 +
 20 files changed, 353 insertions(+), 4 deletions(-)
 create mode 100755 ansible/scripts/extract_docker_cache.sh
 create mode 100755 ansible/scripts/inject_docker_cache.sh
 create mode 100755 builder/install/ccache.sh

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index 4da23532ec..d13aac626f 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -74,6 +74,27 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up builder ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/builder/.ccache
+          key: builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            builder-ccache-${{ matrix.arch }}-master-
+
+      - name: Setup builder ccache in docker cache
+        if: |
+          (github.event_name != 'pull_request' && matrix.arch != 's390x') ||
+          matrix.arch == 'amd64' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+        with:
+          cache-map: |
+            {
+              "${{ github.workspace }}/builder/.ccache": "/root/.ccache"
+            }
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
@@ -152,18 +173,30 @@ jobs:
           (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
         timeout-minutes: 480
         run: |
+          [ ! -d "${{ github.workspace }}/builder/.ccache" ] && rm -f "${{ github.workspace }}/builder/.ccache"
+          ls -al "${{ github.workspace }}/builder/.ccache" || true
           ansible-playbook \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
+          ls -al "${{ github.workspace }}/builder/.ccache" || true
 
       - name: Destroy VMs
         if: always() && matrix.arch == 's390x'
         run: |
           make -C ansible destroy-vms
 
+      - name: Store artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: builder-${{ matrix.arch }}-logs
+          path: |
+            ${{ github.workspace }}/build_builder_image.log
+
   create-multiarch-manifest:
     needs:
     - build-builder-image
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 3c2f8e4288..76b61f131f 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -45,6 +45,14 @@ jobs:
         with:
           submodules: true
 
+      - name: Set up ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            ccache-${{ matrix.arch }}-master-
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -84,6 +92,10 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 
+      - name: Stats for ccache after build
+        run: |
+          ccache -s "${{ github.workspace }}/.ccache"
+
   build-collector-image-remote-vm:
     name: Build Collector on a remote VM
     runs-on: ubuntu-24.04
@@ -123,6 +135,15 @@ jobs:
           vm-type: rhel-${{ matrix.arch }}
           job-tag: builder
 
+      - name: Set up ccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
+            ccache-${{ matrix.arch }}-master-
+
       - name: Create Build VMs
         run: |
           make -C "${{ github.workspace }}/ansible" create-build-vms
@@ -152,6 +173,7 @@ jobs:
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 
diff --git a/.gitignore b/.gitignore
index 2287cee635..c6e4b82c24 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,7 @@
 .idea/
 .rox/
+.ccache/
+builder/.ccache/
 
 integration-tests/container-logs/
 integration-tests/*.log
diff --git a/Makefile b/Makefile
index 51dbd0c731..70868374bb 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,8 @@ NPROCS ?= $(shell nproc)
 DEV_SSH_SERVER_KEY ?= $(CURDIR)/.collector_dev_ssh_host_ed25519_key
 BUILD_BUILDER_IMAGE ?= false
 
+CCACHE_DIR?=$(CURDIR)/.ccache
+
 export COLLECTOR_VERSION := $(COLLECTOR_TAG)
 
 .PHONY: tag
@@ -25,9 +27,11 @@ container-dockerfile-dev:
 builder:
 ifneq ($(BUILD_BUILDER_IMAGE), false)
 	docker buildx build --load --platform ${PLATFORM} \
+		--build-arg USE_CCACHE="${USE_CCACHE}" \
 		-t quay.io/stackrox-io/collector-builder:$(COLLECTOR_BUILDER_TAG) \
 		-f "$(CURDIR)/builder/Dockerfile" \
-		"$(CURDIR)/builder"
+		"$(CURDIR)/builder" \
+		2>&1 | tee build_builder_image.log
 endif
 
 collector: check-builder
@@ -84,6 +88,7 @@ start-builder: builder teardown-builder
 		--name $(COLLECTOR_BUILDER_NAME) \
 		--pull missing \
 		--platform ${PLATFORM} \
+		-v $(CCACHE_DIR):/root/.ccache \
 		-v $(CURDIR):$(CURDIR) \
 		$(if $(LOCAL_SSH_PORT),-p $(LOCAL_SSH_PORT):22 )\
 		-w $(CURDIR) \
diff --git a/Makefile-constants.mk b/Makefile-constants.mk
index b4b4ebd83a..27c1ef49c5 100644
--- a/Makefile-constants.mk
+++ b/Makefile-constants.mk
@@ -19,3 +19,4 @@ BPF_DEBUG_MODE ?= false
 
 COLLECTOR_BUILD_CONTEXT = collector/
 COLLECTOR_BUILDER_NAME ?= collector_builder_$(HOST_ARCH)
+USE_CCACHE ?= true
diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
index b8cbc14868..39d0c5fbca 100644
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
@@ -5,6 +5,7 @@ display_skipped_hosts = false
 host_key_checking = false
 remote_tmp = /tmp/ansible
 forks = 20
+callbacks_enabled = profile_tasks
 
 [ssh_connection]
 ssh_args = -o StrictHostKeyChecking=no -C -o ControlMaster=auto -o ControlPersist=60s -o ServerAliveInterval=30 -o ServerAliveCountMax=10
diff --git a/ansible/ci-build-builder.yml b/ansible/ci-build-builder.yml
index d91284aaee..6fdb11b8dd 100644
--- a/ansible/ci-build-builder.yml
+++ b/ansible/ci-build-builder.yml
@@ -1,5 +1,5 @@
 ---
-- name: Build and push collector image
+- name: Build and push builder image
   hosts: "{{ build_hosts | default('all') }}"
 
   environment:
@@ -10,6 +10,10 @@
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: "builder/.ccache"
+    ccache_archive: "docker.tar.gz"
+    ccache_path: "{{ ccache_dir }}/{{ ccache_archive }}"
+    container_ccache_dir: "/root/.ccache"
 
   tasks:
     - name: Clone repository
@@ -21,13 +25,55 @@
         # than with commit hashes, prevents "reference is not a tree" errors
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
+        depth: 1
         recursive: true
       when: arch == "s390x"
 
+    - name: Check if ccache archive exists
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_path }}"
+      register: ccache_check
+      when: arch == "s390x"
+
+    - name: Copy ccache to build
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_path }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}/"
+      when: arch == "s390x" and ccache_check.stat.exists
+
+    - name: Inject docker cache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/inject_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x" and ccache_check.stat.exists
+
     - name: Build the collector builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: builder
+        params:
+          USE_CCACHE: "true"
+
+    - name: Extract docker cache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/extract_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Fetch ccache
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_path }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: arch == "s390x"
+
+    - name: Fetch build log
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/build_builder_image.log"
+        dest: "{{ github_workspace }}/"
+        flat: yes
+      when: arch == "s390x"
 
     - name: Retag collector builder image to arch specific
       community.docker.docker_image:
diff --git a/ansible/ci-build-collector.yml b/ansible/ci-build-collector.yml
index 960eb4d8f2..a26b09a66a 100644
--- a/ansible/ci-build-collector.yml
+++ b/ansible/ci-build-collector.yml
@@ -12,6 +12,8 @@
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: ".ccache"
+    ccache_archive: "docker.tar.gz"
 
   tasks:
     - debug: var=collector_root
@@ -25,8 +27,29 @@
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
         recursive: true
+        depth: 1
       when: arch == "s390x"
 
+    - name: Check if remote ccache archive exists on controller
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+      register: ccache_check
+      when: arch == "s390x"
+
+    - name: Copy ccache from the controller to build VM
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/"
+      when: arch == "s390x" and ccache_check.stat.exists
+
+    - name: Extract ccache archive if found
+      ansible.builtin.unarchive:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}"
+        remote_src: true
+      when: arch == "s390x" and ccache_check.stat.exists
+
     - name: Run the builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
@@ -36,10 +59,27 @@
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: image
+        params:
+          CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
+          USE_CCACHE: true
       register: build_result
       # ensure this action is printed
       tags: [print_action]
 
+    - name: Create ccache archive
+      ansible.builtin.shell: |
+        rm -f "{{ collector_root }}/{{ ccache_archive }}"
+        cd "{{ collector_root }}/{{ ccache_dir }}"
+        tar czf "{{ collector_root }}/{{ ccache_archive }}" .
+      when: arch == "s390x"
+
+    - name: Copy ccache from build machine to controller
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: arch == "s390x"
+
     - name: Retag collector image to arch specific
       community.docker.docker_image:
         name: "{{ collector_image }}"
diff --git a/ansible/scripts/extract_docker_cache.sh b/ansible/scripts/extract_docker_cache.sh
new file mode 100755
index 0000000000..d7c27ca536
--- /dev/null
+++ b/ansible/scripts/extract_docker_cache.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+# based on https://github.com/reproducible-containers/buildkit-cache-dance
+
+set -e
+
+usage() {
+    echo "Usage: $0 <cache_destination_directory> <target_path_inside_container>"
+    exit 1
+}
+
+if [ "$#" -ne 2 ]; then
+    usage
+fi
+
+CACHE_DEST="$1"
+TARGET_PATH="$2"
+
+CCACHE_ARCHIVE="docker.tar.gz"
+
+SCRATCH_DIR=$(mktemp -d)
+SCRATCH_ARCHIVE=$(mktemp)
+# shellcheck disable=SC2064
+trap "rm -rf ${SCRATCH_DIR}; rm -f ${SCRATCH_ARCHIVE}" EXIT
+
+# Timestamp to bust cache
+date -Iseconds > "${SCRATCH_DIR}/buildstamp"
+
+cat << EOF > "${SCRATCH_DIR}/Dockerfile.extract"
+FROM busybox:1
+COPY buildstamp buildstamp
+RUN --mount=type=cache,target=${TARGET_PATH} \\
+    mkdir -p /var/docker-cache/ \\
+    && cp -p -R ${TARGET_PATH}/. /var/docker-cache/ || true
+EOF
+
+echo "Generated Dockerfile.extract"
+cat "${SCRATCH_DIR}/Dockerfile.extract"
+
+# Build the image and load it into Docker
+docker buildx build -f "${SCRATCH_DIR}/Dockerfile.extract" --tag cache:extract --load "${SCRATCH_DIR}"
+docker images
+
+# Remove any existing cache-container
+docker rm -f cache-container || true
+
+# Create a container from cache:extract
+docker create --name cache-container cache:extract
+docker ps
+
+# Extract the cache from the container
+docker cp -L cache-container:/var/docker-cache - | tar -H posix -x -C "${SCRATCH_DIR}"
+ls "${SCRATCH_DIR}"
+
+# Compress the cache from the container
+(cd "${SCRATCH_DIR}/docker-cache" && chmod -R 777 . && tar czf "${SCRATCH_ARCHIVE}" .)
+
+# Move the cache into its dest
+rm -f "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+mv "${SCRATCH_ARCHIVE}" "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+chmod 666 "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+
+echo "Docker cache extraction completed successfully."
diff --git a/ansible/scripts/inject_docker_cache.sh b/ansible/scripts/inject_docker_cache.sh
new file mode 100755
index 0000000000..2b74bbca85
--- /dev/null
+++ b/ansible/scripts/inject_docker_cache.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# based on https://github.com/reproducible-containers/buildkit-cache-dance
+
+set -e
+
+usage() {
+    echo "Usage: $0 <cache_source_directory> <target_path_inside_container>"
+    exit 1
+}
+
+if [ "$#" -ne 2 ]; then
+    usage
+fi
+
+CACHE_SOURCE="$1"
+TARGET_PATH="$2"
+
+if [ ! -d "$CACHE_SOURCE" ]; then
+    echo "Error: Cache source directory '$CACHE_SOURCE' does not exist."
+    exit 1
+fi
+
+CCACHE_ARCHIVE="docker.tar.gz"
+
+SCRATCH_DIR=$(mktemp -d)
+# shellcheck disable=SC2064
+trap "rm -rf ${SCRATCH_DIR}" EXIT
+
+# Timestamp to bust cache
+date -Iseconds > "${CACHE_SOURCE}/buildstamp"
+
+cat << EOF > "${SCRATCH_DIR}/Dockerfile.inject"
+FROM busybox:1
+COPY buildstamp buildstamp
+RUN --mount=type=cache,target=${TARGET_PATH} \\
+    --mount=type=bind,source=.,target=/var/docker-cache \\
+    tar -xzf /var/docker-cache/${CCACHE_ARCHIVE} -C ${TARGET_PATH} || true
+EOF
+
+echo "Generated Dockerfile.inject"
+cat "${SCRATCH_DIR}/Dockerfile.inject"
+
+# Inject ccache into Docker cache
+cd "$CACHE_SOURCE"
+docker buildx build -f "${SCRATCH_DIR}/Dockerfile.inject" --tag cache:inject .
+
+echo "Cache injection completed successfully."
diff --git a/builder/Dockerfile b/builder/Dockerfile
index 4aee6a0c9c..e143d4dda2 100644
--- a/builder/Dockerfile
+++ b/builder/Dockerfile
@@ -1,9 +1,13 @@
 FROM quay.io/centos/centos:stream9
 
 ARG BUILD_DIR=/install-tmp
+ARG USE_CCACHE=true
+ENV USE_CCACHE=${USE_CCACHE}
 
 USER root
 
+RUN echo $USE_CCACHE
+
 RUN dnf -y update \
     && dnf -y install --nobest \
         autoconf \
@@ -53,7 +57,7 @@ WORKDIR ${BUILD_DIR}
 COPY install builder/install
 COPY third_party third_party
 
-RUN "builder/install/install-dependencies.sh" && \
+RUN --mount=type=cache,target=/root/.ccache/ "builder/install/install-dependencies.sh" && \
     rm -rf ${BUILD_DIR} && \
     echo -e '/usr/local/lib\n/usr/local/lib64' > /etc/ld.so.conf.d/usrlocallib.conf && ldconfig
 
diff --git a/builder/install/10-gperftools.sh b/builder/install/10-gperftools.sh
index 3a6ead8934..032d54f5d9 100755
--- a/builder/install/10-gperftools.sh
+++ b/builder/install/10-gperftools.sh
@@ -2,6 +2,11 @@
 
 set -e
 
+if [ "${USE_CCACHE}" = "true" ]; then
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+fi
+
 cd third_party/gperftools
 cp COPYING "${LICENSE_DIR}/gperftools-${GPERFTOOLS_VERSION}"
 
diff --git a/builder/install/50-jq.sh b/builder/install/50-jq.sh
index b5b27f46e8..6d0fab0e6a 100755
--- a/builder/install/50-jq.sh
+++ b/builder/install/50-jq.sh
@@ -7,6 +7,11 @@ if [ -n "${WITH_RHEL_RPMS}" ]; then
     exit 0
 fi
 
+if [ "${USE_CCACHE}" = "true" ]; then
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+fi
+
 wget "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-${JQ_VERSION}.tar.gz"
 tar -zxf "jq-${JQ_VERSION}.tar.gz"
 cd "jq-${JQ_VERSION}"
diff --git a/builder/install/50-libb64.sh b/builder/install/50-libb64.sh
index f731db8534..f2932127c7 100755
--- a/builder/install/50-libb64.sh
+++ b/builder/install/50-libb64.sh
@@ -8,5 +8,10 @@ cat AUTHORS LICENSE > "${LICENSE_DIR}/libb64-${B64_VERSION}"
 
 CFLAGS=-fPIC make all_base64
 
+if [ "${USE_CCACHE}" = "true" ]; then
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+fi
+
 cp src/libb64.a /usr/local/lib/
 cp -r include/b64 /usr/local/include/
diff --git a/builder/install/60-tbb.sh b/builder/install/60-tbb.sh
index 2bb8288c2a..185f7efbb6 100755
--- a/builder/install/60-tbb.sh
+++ b/builder/install/60-tbb.sh
@@ -7,6 +7,11 @@ if [ -n "${WITH_RHEL_RPMS}" ]; then
     exit 0
 fi
 
+if [ "${USE_CCACHE}" = "true" ]; then
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+fi
+
 git clone --branch "$TBB_VERSION" --depth 1 https://github.com/intel/tbb
 cd tbb
 cp LICENSE "${LICENSE_DIR}/tbb-${TBB_VERSION}"
diff --git a/builder/install/80-libbpf.sh b/builder/install/80-libbpf.sh
index 29a3202cc1..acbab402d0 100755
--- a/builder/install/80-libbpf.sh
+++ b/builder/install/80-libbpf.sh
@@ -2,6 +2,11 @@
 
 set -e
 
+if [ "${USE_CCACHE}" = "true" ]; then
+    export CC="ccache gcc"
+    export CXX="ccache g++"
+fi
+
 cd third_party/libbpf
 
 cp LICENSE "${LICENSE_DIR}/libbpf-${LIBBPF_VERSION}"
diff --git a/builder/install/ccache.sh b/builder/install/ccache.sh
new file mode 100755
index 0000000000..f91256abdf
--- /dev/null
+++ b/builder/install/ccache.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+## TODO: update builder image to fedora and use builtin ccache
+
+if [ "$USE_CCACHE" = "true" ]; then
+
+    git clone https://github.com/ccache/ccache.git third_party/ccache
+
+    cd third_party/ccache
+    git fetch --tags
+    git checkout "$(git describe --tags "$(git rev-list --tags --max-count=1)")"
+
+    mkdir cmake-build && cd cmake-build
+    cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=OFF -DENABLE_DOCUMENTATION=OFF -DREDIS_STORAGE_BACKEND=OFF ..
+
+    make -j "${NPROCS}"
+    make install
+
+    ccache --version
+    ccache -s
+fi
diff --git a/builder/install/install-dependencies.sh b/builder/install/install-dependencies.sh
index 082f62ec88..0073909fb4 100755
--- a/builder/install/install-dependencies.sh
+++ b/builder/install/install-dependencies.sh
@@ -6,8 +6,32 @@ export LICENSE_DIR="/THIRD_PARTY_NOTICES"
 
 mkdir -p "${LICENSE_DIR}"
 
-export NPROCS
 NPROCS="$(nproc)"
+export NPROCS
+
+if [ "${USE_CCACHE}" = "true" ]; then
+    # build and install ccache
+    ./builder/install/ccache.sh
+
+    # create a wrapper utility for clang-17 used by modern bpf builds in falco
+    printf '#!/bin/sh\nexec ccache /usr/bin/clang-17 "$@"\n' > /usr/local/bin/ccache-clang
+    chmod +x /usr/local/bin/ccache-clang
+    echo /usr/local/bin/ccache-clang
+
+    # Use ccache in cmake builds
+    export CMAKE_C_COMPILER_LAUNCHER=ccache
+    export CMAKE_CXX_COMPILER_LAUNCHER=ccache
+
+    # print stats and zero them
+    ccache -z -d /root/.ccache
+
+    #export CCACHE_NOINODECACHE=true
+    #export CCACHE_SLOPPINESS=time_macros,file_macro,include_file_mtime,include_file_ctime,gcno_cwd
+    #export CCACHE_DEBUG=1
+    #export CCACHE_DEBUGDIR=/ccache-debug
+    #export CCACHE_LOGFILE=/ccache.log
+    #mkdir "${CCACHE_DEBUGDIR}" || true
+fi
 
 # shellcheck source=SCRIPTDIR/versions.sh
 source builder/install/versions.sh
@@ -16,3 +40,8 @@ for f in builder/install/[0-9][0-9]-*.sh; do
     ./"$f"
     ldconfig
 done
+
+if [ "${USE_CCACHE}" = "true" ]; then
+    echo "=== ccache stats after build ==="
+    ccache -s -d /root/.ccache
+fi
diff --git a/collector/CMakeLists.txt b/collector/CMakeLists.txt
index bb35c5dc0b..44533ce138 100644
--- a/collector/CMakeLists.txt
+++ b/collector/CMakeLists.txt
@@ -35,6 +35,11 @@ if (NOT COLLECTOR_VERSION)
 	set(COLLECTOR_VERSION "0.0.0")
 endif()
 
+if(USE_CCACHE)
+    set(CMAKE_C_COMPILER_LAUNCHER ccache)
+    set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
+endif()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/lib/CollectorVersion.h.in ${CMAKE_CURRENT_BINARY_DIR}/CollectorVersion.h)
 
 set(FALCO_DIR ${PROJECT_SOURCE_DIR}/../falcosecurity-libs)
@@ -85,6 +90,9 @@ add_executable(self-checks self-checks.cpp)
 add_subdirectory(test)
 
 # Falco Wrapper Library
+if(USE_CCACHE)
+  set(MODERN_CLANG_EXE "/usr/local/bin/ccache-clang" CACHE STRING "USE CCACHE" FORCE)
+endif()
 set(BUILD_DRIVER OFF CACHE BOOL "Build the driver on Linux" FORCE)
 set(USE_BUNDLED_DEPS OFF CACHE BOOL "Enable bundled dependencies instead of using the system ones" FORCE)
 set(USE_BUNDLED_CARES OFF CACHE BOOL "Enable bundled dependencies instead of using the system ones" FORCE)
diff --git a/collector/Makefile b/collector/Makefile
index 0295a8d26d..6985b4d72f 100644
--- a/collector/Makefile
+++ b/collector/Makefile
@@ -20,6 +20,7 @@ COLLECTOR_BUILD_DEPS := $(HDRS) $(SRCS) $(shell find $(BASE_PATH)/falcosecurity-
 cmake-configure/collector:
 	docker exec $(COLLECTOR_BUILDER_NAME) \
 		cmake -S $(BASE_PATH) -B $(CMAKE_DIR) \
+			-DUSE_CCACHE=$(USE_CCACHE)
 			-DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) \
 			-DDISABLE_PROFILING=$(DISABLE_PROFILING) \
 			-DUSE_VALGRIND=$(USE_VALGRIND) \

From 5479a24af105d08ae1a47cc1095665c5834fe8aa Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sat, 14 Sep 2024 14:54:06 -0700
Subject: [PATCH 03/12] create ccache dir on remote for collector build
 optimize git on s390x

---
 ansible/ci-build-builder.yml   | 17 ++++++++++++-----
 ansible/ci-build-collector.yml | 23 ++++++++++++++++++-----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/ansible/ci-build-builder.yml b/ansible/ci-build-builder.yml
index 6fdb11b8dd..087f3a79ea 100644
--- a/ansible/ci-build-builder.yml
+++ b/ansible/ci-build-builder.yml
@@ -26,23 +26,30 @@
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
         depth: 1
-        recursive: true
+        recursive: false
       when: arch == "s390x"
 
-    - name: Check if ccache archive exists
+    - name: Clone submodules
+      ansible.builtin.shell: |
+        git submodule update --init --depth 1
+      args:
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Check if ccache exists
       delegate_to: localhost
       ansible.builtin.stat:
         path: "{{ github_workspace }}/{{ ccache_path }}"
       register: ccache_check
       when: arch == "s390x"
 
-    - name: Copy ccache to build
+    - name: Copy ccache
       ansible.builtin.copy:
         src: "{{ github_workspace }}/{{ ccache_path }}"
         dest: "{{ collector_root }}/{{ ccache_dir }}/"
       when: arch == "s390x" and ccache_check.stat.exists
 
-    - name: Inject docker cache
+    - name: Inject docker cache with ccache
       ansible.builtin.shell:
         cmd: ansible/scripts/inject_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
         chdir: "{{ collector_root }}"
@@ -55,7 +62,7 @@
         params:
           USE_CCACHE: "true"
 
-    - name: Extract docker cache
+    - name: Extract ccache from docker cache
       ansible.builtin.shell:
         cmd: ansible/scripts/extract_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
         chdir: "{{ collector_root }}"
diff --git a/ansible/ci-build-collector.yml b/ansible/ci-build-collector.yml
index a26b09a66a..10a840b1a8 100644
--- a/ansible/ci-build-collector.yml
+++ b/ansible/ci-build-collector.yml
@@ -26,24 +26,37 @@
         # than with commit hashes, prevents "reference is not a tree" errors
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
-        recursive: true
         depth: 1
+        recursive: false
       when: arch == "s390x"
 
-    - name: Check if remote ccache archive exists on controller
+    - name: Clone submodules
+      ansible.builtin.shell: |
+        git submodule update --init --depth 1
+      args:
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Check if ccache exists
       delegate_to: localhost
       ansible.builtin.stat:
         path: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
       register: ccache_check
       when: arch == "s390x"
 
-    - name: Copy ccache from the controller to build VM
+    - name: Create cccache directory
+      ansible.builtin.file:
+        path: "{{ collector_root }}/{{ ccache_dir }}"
+        state: directory
+      when: arch == "s390x" and ccache_check.stat.exists
+
+    - name: Copy ccache
       ansible.builtin.copy:
         src: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
         dest: "{{ collector_root }}/"
       when: arch == "s390x" and ccache_check.stat.exists
 
-    - name: Extract ccache archive if found
+    - name: Unarchive ccache
       ansible.builtin.unarchive:
         src: "{{ collector_root }}/{{ ccache_archive }}"
         dest: "{{ collector_root }}/{{ ccache_dir }}"
@@ -73,7 +86,7 @@
         tar czf "{{ collector_root }}/{{ ccache_archive }}" .
       when: arch == "s390x"
 
-    - name: Copy ccache from build machine to controller
+    - name: Fetch ccache
       ansible.builtin.fetch:
         src: "{{ collector_root }}/{{ ccache_archive }}"
         dest: "{{ github_workspace }}/{{ ccache_dir }}/"

From 6366dc32c9fa5868062a347855275caf512cabb7 Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sun, 15 Sep 2024 06:04:43 -0700
Subject: [PATCH 04/12] key names

---
 .github/workflows/collector-builder.yml |  8 ++------
 .github/workflows/collector.yml         | 14 +++-----------
 2 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index d13aac626f..e03341b766 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -78,10 +78,9 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/builder/.ccache
-          key: builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          key: builder-ccache-${{ matrix.arch }}-${{ format('YYYY-MM-DDTHH:mm:ssZ', github.event.head_commit.timestamp) }}
           restore-keys: |
-            builder-ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
-            builder-ccache-${{ matrix.arch }}-master-
+            builder-ccache-${{ matrix.arch }}-
 
       - name: Setup builder ccache in docker cache
         if: |
@@ -173,8 +172,6 @@ jobs:
           (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
         timeout-minutes: 480
         run: |
-          [ ! -d "${{ github.workspace }}/builder/.ccache" ] && rm -f "${{ github.workspace }}/builder/.ccache"
-          ls -al "${{ github.workspace }}/builder/.ccache" || true
           ansible-playbook \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
@@ -182,7 +179,6 @@ jobs:
             -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
-          ls -al "${{ github.workspace }}/builder/.ccache" || true
 
       - name: Destroy VMs
         if: always() && matrix.arch == 's390x'
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 76b61f131f..884fe09799 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -48,10 +48,7 @@ jobs:
       - name: Set up ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
-          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
-          restore-keys: |
-            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
-            ccache-${{ matrix.arch }}-master-
+          key: ${{ matrix.arch }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -92,10 +89,6 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 
-      - name: Stats for ccache after build
-        run: |
-          ccache -s "${{ github.workspace }}/.ccache"
-
   build-collector-image-remote-vm:
     name: Build Collector on a remote VM
     runs-on: ubuntu-24.04
@@ -139,10 +132,9 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/.ccache
-          key: ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }}
+          key: ccache-${{ matrix.arch }}-${{ format('YYYY-MM-DDTHH:mm:ssZ', github.event.head_commit.timestamp) }}
           restore-keys: |
-            ccache-${{ matrix.arch }}-${{ github.head_ref || github.ref_name }}-
-            ccache-${{ matrix.arch }}-master-
+            ccache-${{ matrix.arch }}-
 
       - name: Create Build VMs
         run: |

From 1693602693925b25517e190d6b2eba8589518785 Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sun, 15 Sep 2024 06:31:54 -0700
Subject: [PATCH 05/12] fmt

---
 .github/workflows/collector-builder.yml | 9 +++------
 .github/workflows/collector.yml         | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index e03341b766..55e2b516bb 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -78,15 +78,12 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/builder/.ccache
-          key: builder-ccache-${{ matrix.arch }}-${{ format('YYYY-MM-DDTHH:mm:ssZ', github.event.head_commit.timestamp) }}
+          key: builder-ccache-${{ matrix.arch }}-${{ github.event.head_commit.timestamp }}
           restore-keys: |
             builder-ccache-${{ matrix.arch }}-
 
-      - name: Setup builder ccache in docker cache
-        if: |
-          (github.event_name != 'pull_request' && matrix.arch != 's390x') ||
-          matrix.arch == 'amd64' ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
+      - name: Set up builder ccache in docker cache
+        if: matrix.arch != 's390x'
         uses: reproducible-containers/buildkit-cache-dance@v3.1.2
         with:
           cache-map: |
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 884fe09799..d1114fcfea 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -132,7 +132,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/.ccache
-          key: ccache-${{ matrix.arch }}-${{ format('YYYY-MM-DDTHH:mm:ssZ', github.event.head_commit.timestamp) }}
+          key: ccache-${{ matrix.arch }}-${{ github.event.head_commit.timestamp }}
           restore-keys: |
             ccache-${{ matrix.arch }}-
 

From b8980b715f019b978fee25527b94c72ddce728ac Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sun, 15 Sep 2024 12:54:04 -0700
Subject: [PATCH 06/12] key

---
 .github/workflows/collector-builder.yml | 2 +-
 .github/workflows/collector.yml         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index 55e2b516bb..db215c98ca 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -78,7 +78,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/builder/.ccache
-          key: builder-ccache-${{ matrix.arch }}-${{ github.event.head_commit.timestamp }}
+          key: builder-ccache-${{ matrix.arch }}-${{ github.sha }}
           restore-keys: |
             builder-ccache-${{ matrix.arch }}-
 
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index d1114fcfea..1168f4dce1 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -132,7 +132,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/.ccache
-          key: ccache-${{ matrix.arch }}-${{ github.event.head_commit.timestamp }}
+          key: ccache-${{ matrix.arch }}-${{ github.sha }}
           restore-keys: |
             ccache-${{ matrix.arch }}-
 

From 770f8c3d566f046d6f2178ae0e2b96bd3ffacecd Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sun, 15 Sep 2024 22:08:01 -0700
Subject: [PATCH 07/12] label

---
 .github/workflows/collector-builder.yml | 10 ++--------
 .github/workflows/collector.yml         |  3 +++
 Makefile                                |  3 +--
 Makefile-constants.mk                   |  2 +-
 ansible/ci-build-builder.yml            | 20 ++++++--------------
 ansible/ci-build-collector.yml          | 17 ++++++++---------
 builder/Dockerfile                      |  4 +---
 builder/install/ccache.sh               |  4 +---
 builder/install/install-dependencies.sh |  8 --------
 builder/install/versions.sh             |  1 +
 10 files changed, 24 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index db215c98ca..6ac491a149 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -17,6 +17,7 @@ env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
   DEFAULT_BUILDER_TAG: master
   ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
+  USE_CCACHE: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ccache') }}
 
 jobs:
   builder-needs-rebuilding:
@@ -145,6 +146,7 @@ jobs:
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
             echo "collector_git_ref: ${{ github.ref }}"
             echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
+            echo "use_ccache: ${USE_CCACHE}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
       - name: Build images
@@ -182,14 +184,6 @@ jobs:
         run: |
           make -C ansible destroy-vms
 
-      - name: Store artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: builder-${{ matrix.arch }}-logs
-          path: |
-            ${{ github.workspace }}/build_builder_image.log
-
   create-multiarch-manifest:
     needs:
     - build-builder-image
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 1168f4dce1..4868dd54af 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -26,6 +26,7 @@ env:
   ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
   TRACE_SINSP_EVENTS: ${{ github.event_name == 'pull_request' }}
   ADDRESS_SANITIZER: ${{ contains(github.event.pull_request.labels.*.name, 'address-sanitizer') }}
+  USE_CCACHE: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ccache') }}
 
 jobs:
   build-collector-image:
@@ -72,6 +73,7 @@ jobs:
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
           driver_version: ${DRIVER_VERSION}
+          use_ccache: ${USE_CCACHE}
           EOF
 
       - name: Build images
@@ -156,6 +158,7 @@ jobs:
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
+          use_ccache: ${USE_CCACHE}
           EOF
 
       - name: Build ${{ matrix.arch }} image
diff --git a/Makefile b/Makefile
index 70868374bb..bd0008af6e 100644
--- a/Makefile
+++ b/Makefile
@@ -30,8 +30,7 @@ ifneq ($(BUILD_BUILDER_IMAGE), false)
 		--build-arg USE_CCACHE="${USE_CCACHE}" \
 		-t quay.io/stackrox-io/collector-builder:$(COLLECTOR_BUILDER_TAG) \
 		-f "$(CURDIR)/builder/Dockerfile" \
-		"$(CURDIR)/builder" \
-		2>&1 | tee build_builder_image.log
+		"$(CURDIR)/builder"
 endif
 
 collector: check-builder
diff --git a/Makefile-constants.mk b/Makefile-constants.mk
index 27c1ef49c5..bc61d5f7be 100644
--- a/Makefile-constants.mk
+++ b/Makefile-constants.mk
@@ -10,6 +10,7 @@ HOST_ARCH := $(shell uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')
 PLATFORM ?= "linux/$(HOST_ARCH)"
 
 USE_VALGRIND ?= false
+USE_CCACHE ?= false
 ADDRESS_SANITIZER ?= false
 CMAKE_BUILD_TYPE ?= Release
 CMAKE_BASE_DIR = cmake-build-$(shell echo $(CMAKE_BUILD_TYPE) | tr A-Z a-z)-$(HOST_ARCH)
@@ -19,4 +20,3 @@ BPF_DEBUG_MODE ?= false
 
 COLLECTOR_BUILD_CONTEXT = collector/
 COLLECTOR_BUILDER_NAME ?= collector_builder_$(HOST_ARCH)
-USE_CCACHE ?= true
diff --git a/ansible/ci-build-builder.yml b/ansible/ci-build-builder.yml
index 087f3a79ea..462c00279e 100644
--- a/ansible/ci-build-builder.yml
+++ b/ansible/ci-build-builder.yml
@@ -6,6 +6,7 @@
     BUILD_BUILDER_IMAGE: "true"
     COLLECTOR_BUILDER_TAG: "{{ collector_builder_tag }}"
     PLATFORM: "linux/{{ arch }}"
+    USE_CCACHE: "{{ use_ccache }}"
 
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
@@ -41,46 +42,37 @@
       ansible.builtin.stat:
         path: "{{ github_workspace }}/{{ ccache_path }}"
       register: ccache_check
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Copy ccache
       ansible.builtin.copy:
         src: "{{ github_workspace }}/{{ ccache_path }}"
         dest: "{{ collector_root }}/{{ ccache_dir }}/"
-      when: arch == "s390x" and ccache_check.stat.exists
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
 
     - name: Inject docker cache with ccache
       ansible.builtin.shell:
         cmd: ansible/scripts/inject_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
         chdir: "{{ collector_root }}"
-      when: arch == "s390x" and ccache_check.stat.exists
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
 
     - name: Build the collector builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: builder
-        params:
-          USE_CCACHE: "true"
 
     - name: Extract ccache from docker cache
       ansible.builtin.shell:
         cmd: ansible/scripts/extract_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
         chdir: "{{ collector_root }}"
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Fetch ccache
       ansible.builtin.fetch:
         src: "{{ collector_root }}/{{ ccache_path }}"
         dest: "{{ github_workspace }}/{{ ccache_dir }}/"
         flat: yes
-      when: arch == "s390x"
-
-    - name: Fetch build log
-      ansible.builtin.fetch:
-        src: "{{ collector_root }}/build_builder_image.log"
-        dest: "{{ github_workspace }}/"
-        flat: yes
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Retag collector builder image to arch specific
       community.docker.docker_image:
diff --git a/ansible/ci-build-collector.yml b/ansible/ci-build-collector.yml
index 10a840b1a8..1ecd76243c 100644
--- a/ansible/ci-build-collector.yml
+++ b/ansible/ci-build-collector.yml
@@ -8,6 +8,8 @@
     COLLECTOR_TAG: "{{ collector_tag }}"
     DISABLE_PROFILING: "{{ disable_profiling }}"
     CMAKE_BUILD_TYPE: "{{ 'Debug' if debug_mode else 'Release' }}"
+    USE_CCACHE: "{{ use_ccache }}"
+    CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
 
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
@@ -42,26 +44,26 @@
       ansible.builtin.stat:
         path: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
       register: ccache_check
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Create cccache directory
       ansible.builtin.file:
         path: "{{ collector_root }}/{{ ccache_dir }}"
         state: directory
-      when: arch == "s390x" and ccache_check.stat.exists
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
 
     - name: Copy ccache
       ansible.builtin.copy:
         src: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
         dest: "{{ collector_root }}/"
-      when: arch == "s390x" and ccache_check.stat.exists
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
 
     - name: Unarchive ccache
       ansible.builtin.unarchive:
         src: "{{ collector_root }}/{{ ccache_archive }}"
         dest: "{{ collector_root }}/{{ ccache_dir }}"
         remote_src: true
-      when: arch == "s390x" and ccache_check.stat.exists
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
 
     - name: Run the builder image
       community.general.make:
@@ -72,9 +74,6 @@
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: image
-        params:
-          CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
-          USE_CCACHE: true
       register: build_result
       # ensure this action is printed
       tags: [print_action]
@@ -84,14 +83,14 @@
         rm -f "{{ collector_root }}/{{ ccache_archive }}"
         cd "{{ collector_root }}/{{ ccache_dir }}"
         tar czf "{{ collector_root }}/{{ ccache_archive }}" .
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Fetch ccache
       ansible.builtin.fetch:
         src: "{{ collector_root }}/{{ ccache_archive }}"
         dest: "{{ github_workspace }}/{{ ccache_dir }}/"
         flat: yes
-      when: arch == "s390x"
+      when: use_ccache and arch == "s390x"
 
     - name: Retag collector image to arch specific
       community.docker.docker_image:
diff --git a/builder/Dockerfile b/builder/Dockerfile
index e143d4dda2..9a88cb11e0 100644
--- a/builder/Dockerfile
+++ b/builder/Dockerfile
@@ -1,13 +1,11 @@
 FROM quay.io/centos/centos:stream9
 
 ARG BUILD_DIR=/install-tmp
-ARG USE_CCACHE=true
+ARG USE_CCACHE=false
 ENV USE_CCACHE=${USE_CCACHE}
 
 USER root
 
-RUN echo $USE_CCACHE
-
 RUN dnf -y update \
     && dnf -y install --nobest \
         autoconf \
diff --git a/builder/install/ccache.sh b/builder/install/ccache.sh
index f91256abdf..6720f0860c 100755
--- a/builder/install/ccache.sh
+++ b/builder/install/ccache.sh
@@ -10,14 +10,12 @@ if [ "$USE_CCACHE" = "true" ]; then
 
     cd third_party/ccache
     git fetch --tags
-    git checkout "$(git describe --tags "$(git rev-list --tags --max-count=1)")"
+    git checkout "${CCACHE_VERSION}"
 
     mkdir cmake-build && cd cmake-build
     cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=OFF -DENABLE_DOCUMENTATION=OFF -DREDIS_STORAGE_BACKEND=OFF ..
 
     make -j "${NPROCS}"
     make install
-
     ccache --version
-    ccache -s
 fi
diff --git a/builder/install/install-dependencies.sh b/builder/install/install-dependencies.sh
index 0073909fb4..dd7dbe60c2 100755
--- a/builder/install/install-dependencies.sh
+++ b/builder/install/install-dependencies.sh
@@ -24,13 +24,6 @@ if [ "${USE_CCACHE}" = "true" ]; then
 
     # print stats and zero them
     ccache -z -d /root/.ccache
-
-    #export CCACHE_NOINODECACHE=true
-    #export CCACHE_SLOPPINESS=time_macros,file_macro,include_file_mtime,include_file_ctime,gcno_cwd
-    #export CCACHE_DEBUG=1
-    #export CCACHE_DEBUGDIR=/ccache-debug
-    #export CCACHE_LOGFILE=/ccache.log
-    #mkdir "${CCACHE_DEBUGDIR}" || true
 fi
 
 # shellcheck source=SCRIPTDIR/versions.sh
@@ -42,6 +35,5 @@ for f in builder/install/[0-9][0-9]-*.sh; do
 done
 
 if [ "${USE_CCACHE}" = "true" ]; then
-    echo "=== ccache stats after build ==="
     ccache -s -d /root/.ccache
 fi
diff --git a/builder/install/versions.sh b/builder/install/versions.sh
index 075f8bd3f0..93dcd87808 100644
--- a/builder/install/versions.sh
+++ b/builder/install/versions.sh
@@ -2,6 +2,7 @@
 
 export B64_VERSION=1.2.1
 export CARES_VERSION=1.16.0
+export CCACHE_VERSION=v4.10.2
 export CMAKE_VERSION=3.15.2
 export GOOGLETEST_REVISION=release-1.10.0
 export GRPC_REVISION=v1.28.1

From 72a92576e9028d92f5c028a734bbd273fc22797e Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Sun, 15 Sep 2024 23:10:00 -0700
Subject: [PATCH 08/12] fmt

---
 Makefile           | 2 +-
 builder/Dockerfile | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index bd0008af6e..94c61a50a6 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ container-dockerfile-dev:
 builder:
 ifneq ($(BUILD_BUILDER_IMAGE), false)
 	docker buildx build --load --platform ${PLATFORM} \
-		--build-arg USE_CCACHE="${USE_CCACHE}" \
+		--build-arg USE_CCACHE="$(USE_CCACHE)" \
 		-t quay.io/stackrox-io/collector-builder:$(COLLECTOR_BUILDER_TAG) \
 		-f "$(CURDIR)/builder/Dockerfile" \
 		"$(CURDIR)/builder"
diff --git a/builder/Dockerfile b/builder/Dockerfile
index 9a88cb11e0..68d6306d7b 100644
--- a/builder/Dockerfile
+++ b/builder/Dockerfile
@@ -2,6 +2,7 @@ FROM quay.io/centos/centos:stream9
 
 ARG BUILD_DIR=/install-tmp
 ARG USE_CCACHE=false
+
 ENV USE_CCACHE=${USE_CCACHE}
 
 USER root

From 56fe902216c923fab8b1de0cb0ab64b1f628912f Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Mon, 16 Sep 2024 13:10:15 -0700
Subject: [PATCH 09/12] optional steps

---
 .github/workflows/collector-builder.yml | 5 +++--
 .github/workflows/collector.yml         | 6 ++++--
 builder/install/install-dependencies.sh | 5 +++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index 6ac491a149..37d21d29c5 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -76,6 +76,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Set up builder ccache
+        if: env.USE_CCACHE
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/builder/.ccache
@@ -84,7 +85,7 @@ jobs:
             builder-ccache-${{ matrix.arch }}-
 
       - name: Set up builder ccache in docker cache
-        if: matrix.arch != 's390x'
+        if: env.USE_CCACHE && matrix.arch != 's390x'
         uses: reproducible-containers/buildkit-cache-dance@v3.1.2
         with:
           cache-map: |
@@ -146,7 +147,7 @@ jobs:
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
             echo "collector_git_ref: ${{ github.ref }}"
             echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
-            echo "use_ccache: ${USE_CCACHE}"
+            echo "use_ccache: ${{ env.USE_CCACHE }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
       - name: Build images
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 4868dd54af..412d5a3323 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -47,6 +47,7 @@ jobs:
           submodules: true
 
       - name: Set up ccache
+        if: env.USE_CCACHE
         uses: hendrikmuhs/ccache-action@v1.2
         with:
           key: ${{ matrix.arch }}
@@ -73,7 +74,7 @@ jobs:
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
           driver_version: ${DRIVER_VERSION}
-          use_ccache: ${USE_CCACHE}
+          use_ccache: ${( env.USE_CCACHE }}
           EOF
 
       - name: Build images
@@ -131,6 +132,7 @@ jobs:
           job-tag: builder
 
       - name: Set up ccache
+        if: env.USE_CCACHE
         uses: actions/cache@v4
         with:
           path: ${{ github.workspace }}/.ccache
@@ -158,7 +160,7 @@ jobs:
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
-          use_ccache: ${USE_CCACHE}
+          use_ccache: ${( env.USE_CCACHE }}
           EOF
 
       - name: Build ${{ matrix.arch }} image
diff --git a/builder/install/install-dependencies.sh b/builder/install/install-dependencies.sh
index dd7dbe60c2..50b64aa6dc 100755
--- a/builder/install/install-dependencies.sh
+++ b/builder/install/install-dependencies.sh
@@ -9,6 +9,9 @@ mkdir -p "${LICENSE_DIR}"
 NPROCS="$(nproc)"
 export NPROCS
 
+# shellcheck source=SCRIPTDIR/versions.sh
+source builder/install/versions.sh
+
 if [ "${USE_CCACHE}" = "true" ]; then
     # build and install ccache
     ./builder/install/ccache.sh
@@ -26,8 +29,6 @@ if [ "${USE_CCACHE}" = "true" ]; then
     ccache -z -d /root/.ccache
 fi
 
-# shellcheck source=SCRIPTDIR/versions.sh
-source builder/install/versions.sh
 for f in builder/install/[0-9][0-9]-*.sh; do
     echo "=== $f ==="
     ./"$f"

From abdf93aaa77ad52877246b258f768e7e11f3c9d2 Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Mon, 16 Sep 2024 13:14:12 -0700
Subject: [PATCH 10/12] debug ansible

---
 .github/workflows/collector-builder.yml | 2 ++
 .github/workflows/collector.yml         | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index 37d21d29c5..88aa8c511a 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -159,6 +159,7 @@ jobs:
         run: |
           ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
+            -vvv \ # DEBUG
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -173,6 +174,7 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
+            -vvv \ # DEBUG
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index 412d5a3323..b01366ee19 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -85,6 +85,7 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
+            -vvv \ # DEBUG
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -167,6 +168,7 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
+            -vvv \ # DEBUG
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \

From 7f134050e3206ecd934506027270a3f2b25f0196 Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Mon, 16 Sep 2024 13:21:20 -0700
Subject: [PATCH 11/12] fmt

---
 .github/workflows/collector-builder.yml | 4 ++--
 .github/workflows/collector.yml         | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index 88aa8c511a..fb92613e0b 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -159,7 +159,7 @@ jobs:
         run: |
           ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
-            -vvv \ # DEBUG
+            -vvv \
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -174,7 +174,7 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \ # DEBUG
+            -vvv \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index b01366ee19..fcc2b08a9f 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -74,7 +74,7 @@ jobs:
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
           driver_version: ${DRIVER_VERSION}
-          use_ccache: ${( env.USE_CCACHE }}
+          use_ccache: ${{ env.USE_CCACHE }}
           EOF
 
       - name: Build images
@@ -85,7 +85,7 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \ # DEBUG
+            -vvv \
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -161,14 +161,14 @@ jobs:
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
-          use_ccache: ${( env.USE_CCACHE }}
+          use_ccache: ${{ env.USE_CCACHE }}
           EOF
 
       - name: Build ${{ matrix.arch }} image
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \ # DEBUG
+            -vvv \
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \

From 4f69acf3e86c1b5eddfa1cb42b43360f721ef92c Mon Sep 17 00:00:00 2001
From: Robby Cochran <rcochran@redhat.com>
Date: Mon, 16 Sep 2024 13:36:27 -0700
Subject: [PATCH 12/12] fix

---
 .github/workflows/collector-builder.yml | 2 --
 .github/workflows/collector.yml         | 2 --
 ansible/ci-build-builder.yml            | 2 +-
 ansible/ci-build-collector.yml          | 2 +-
 4 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml
index fb92613e0b..37d21d29c5 100644
--- a/.github/workflows/collector-builder.yml
+++ b/.github/workflows/collector-builder.yml
@@ -159,7 +159,6 @@ jobs:
         run: |
           ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
-            -vvv \
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -174,7 +173,6 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml
index fcc2b08a9f..1be7dc8d3e 100644
--- a/.github/workflows/collector.yml
+++ b/.github/workflows/collector.yml
@@ -85,7 +85,6 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \
             --connection local \
             -i localhost, \
             --limit localhost \
@@ -168,7 +167,6 @@ jobs:
         timeout-minutes: 480
         run: |
           ansible-playbook \
-            -vvv \
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
diff --git a/ansible/ci-build-builder.yml b/ansible/ci-build-builder.yml
index 462c00279e..443feb018a 100644
--- a/ansible/ci-build-builder.yml
+++ b/ansible/ci-build-builder.yml
@@ -6,7 +6,7 @@
     BUILD_BUILDER_IMAGE: "true"
     COLLECTOR_BUILDER_TAG: "{{ collector_builder_tag }}"
     PLATFORM: "linux/{{ arch }}"
-    USE_CCACHE: "{{ use_ccache }}"
+    USE_CCACHE: "{{ use_ccache|bool|lower }}"
 
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
diff --git a/ansible/ci-build-collector.yml b/ansible/ci-build-collector.yml
index 1ecd76243c..5be8d5ea34 100644
--- a/ansible/ci-build-collector.yml
+++ b/ansible/ci-build-collector.yml
@@ -8,7 +8,7 @@
     COLLECTOR_TAG: "{{ collector_tag }}"
     DISABLE_PROFILING: "{{ disable_profiling }}"
     CMAKE_BUILD_TYPE: "{{ 'Debug' if debug_mode else 'Release' }}"
-    USE_CCACHE: "{{ use_ccache }}"
+    USE_CCACHE: "{{ use_ccache|bool|lower }}"
     CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
 
   vars: