ROX-26272 Enable ccache for faster builds

.github/workflows/collector-builder.yml

@@ -17,6 +17,7 @@ env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
   DEFAULT_BUILDER_TAG: master
   ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
+  USE_CCACHE: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ccache') }}
 
 jobs:
   builder-needs-rebuilding:
@@ -74,6 +75,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up builder ccache
+        if: env.USE_CCACHE
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/builder/.ccache
+          key: builder-ccache-${{ matrix.arch }}-${{ github.sha }}
+          restore-keys: |
+            builder-ccache-${{ matrix.arch }}-
+
+      - name: Set up builder ccache in docker cache
+        if: env.USE_CCACHE && matrix.arch != 's390x'
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.2
+        with:
+          cache-map: |
+            {
+              "${{ github.workspace }}/builder/.ccache": "/root/.ccache"
+            }
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
@@ -128,6 +147,7 @@ jobs:
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
             echo "collector_git_ref: ${{ github.ref }}"
             echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
+            echo "use_ccache: ${{ env.USE_CCACHE }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
       - name: Build images
@@ -156,6 +176,7 @@ jobs:
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e arch='${{ matrix.arch }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
 

.github/workflows/collector.yml

@@ -26,6 +26,7 @@ env:
   ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
   TRACE_SINSP_EVENTS: ${{ github.event_name == 'pull_request' }}
   ADDRESS_SANITIZER: ${{ contains(github.event.pull_request.labels.*.name, 'address-sanitizer') }}
+  USE_CCACHE: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ccache') }}
 
 jobs:
   build-collector-image:
@@ -45,6 +46,12 @@ jobs:
         with:
           submodules: true
 
+      - name: Set up ccache
+        if: env.USE_CCACHE
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ${{ matrix.arch }}
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -67,6 +74,7 @@ jobs:
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
           driver_version: ${DRIVER_VERSION}
+          use_ccache: ${{ env.USE_CCACHE }}
           EOF
 
       - name: Build images
@@ -123,6 +131,15 @@ jobs:
           vm-type: rhel-${{ matrix.arch }}
           job-tag: builder
 
+      - name: Set up ccache
+        if: env.USE_CCACHE
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ccache-${{ matrix.arch }}-${{ github.sha }}
+          restore-keys: |
+            ccache-${{ matrix.arch }}-
+
       - name: Create Build VMs
         run: |
           make -C "${{ github.workspace }}/ansible" create-build-vms
@@ -143,6 +160,7 @@ jobs:
           collector_image: ${{ inputs.collector-image }}
           collector_tag: ${{ inputs.collector-tag }}
           debug_mode: ${{ github.event_name == 'pull_request' }}
+          use_ccache: ${{ env.USE_CCACHE }}
           EOF
 
       - name: Build ${{ matrix.arch }} image
@@ -152,6 +170,7 @@ jobs:
             -i ansible/ci \
             -e arch='${{ matrix.arch }}' \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
+            -e github_workspace='${{ github.workspace }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
 

.gitignore

@@ -1,5 +1,7 @@
 .idea/
 .rox/
+.ccache/
+builder/.ccache/
 
 integration-tests/container-logs/
 integration-tests/*.log

Makefile

@@ -6,6 +6,8 @@ NPROCS ?= $(shell nproc)
 DEV_SSH_SERVER_KEY ?= $(CURDIR)/.collector_dev_ssh_host_ed25519_key
 BUILD_BUILDER_IMAGE ?= false
 
+CCACHE_DIR?=$(CURDIR)/.ccache
+
 export COLLECTOR_VERSION := $(COLLECTOR_TAG)
 
 .PHONY: tag
@@ -25,6 +27,7 @@ container-dockerfile-dev:
 builder:
 ifneq ($(BUILD_BUILDER_IMAGE), false)
 	docker buildx build --load --platform ${PLATFORM} \
+		--build-arg USE_CCACHE="$(USE_CCACHE)" \
 		-t quay.io/stackrox-io/collector-builder:$(COLLECTOR_BUILDER_TAG) \
 		-f "$(CURDIR)/builder/Dockerfile" \
 		"$(CURDIR)/builder"
@@ -84,6 +87,7 @@ start-builder: builder teardown-builder
 		--name $(COLLECTOR_BUILDER_NAME) \
 		--pull missing \
 		--platform ${PLATFORM} \
+		-v $(CCACHE_DIR):/root/.ccache \
 		-v $(CURDIR):$(CURDIR) \
 		$(if $(LOCAL_SSH_PORT),-p $(LOCAL_SSH_PORT):22 )\
 		-w $(CURDIR) \

Makefile-constants.mk

@@ -10,6 +10,7 @@ HOST_ARCH := $(shell uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')
 PLATFORM ?= "linux/$(HOST_ARCH)"
 
 USE_VALGRIND ?= false
+USE_CCACHE ?= false
 ADDRESS_SANITIZER ?= false
 CMAKE_BUILD_TYPE ?= Release
 CMAKE_BASE_DIR = cmake-build-$(shell echo $(CMAKE_BUILD_TYPE) | tr A-Z a-z)-$(HOST_ARCH)

ansible/ansible.cfg

@@ -5,6 +5,7 @@ display_skipped_hosts = false
 host_key_checking = false
 remote_tmp = /tmp/ansible
 forks = 20
+callbacks_enabled = profile_tasks
 
 [ssh_connection]
 ssh_args = -o StrictHostKeyChecking=no -C -o ControlMaster=auto -o ControlPersist=60s -o ServerAliveInterval=30 -o ServerAliveCountMax=10

ansible/ci-build-builder.yml

@@ -1,15 +1,20 @@
 ---
-- name: Build and push collector image
+- name: Build and push builder image
   hosts: "{{ build_hosts | default('all') }}"
 
   environment:
     BUILD_BUILDER_IMAGE: "true"
     COLLECTOR_BUILDER_TAG: "{{ collector_builder_tag }}"
     PLATFORM: "linux/{{ arch }}"
+    USE_CCACHE: "{{ use_ccache|bool|lower }}"
 
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: "builder/.ccache"
+    ccache_archive: "docker.tar.gz"
+    ccache_path: "{{ ccache_dir }}/{{ ccache_archive }}"
+    container_ccache_dir: "/root/.ccache"
 
   tasks:
     - name: Clone repository
@@ -21,14 +26,54 @@
         # than with commit hashes, prevents "reference is not a tree" errors
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
-        recursive: true
+        depth: 1
+        recursive: false
       when: arch == "s390x"
 
+    - name: Clone submodules
+      ansible.builtin.shell: |
+        git submodule update --init --depth 1
+      args:
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Check if ccache exists
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_path }}"
+      register: ccache_check
+      when: use_ccache and arch == "s390x"
+
+    - name: Copy ccache
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_path }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}/"
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
+
+    - name: Inject docker cache with ccache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/inject_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
+
     - name: Build the collector builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: builder
 
+    - name: Extract ccache from docker cache
+      ansible.builtin.shell:
+        cmd: ansible/scripts/extract_docker_cache.sh "{{ collector_root }}/{{ ccache_dir }}" "{{ container_ccache_dir }}"
+        chdir: "{{ collector_root }}"
+      when: use_ccache and arch == "s390x"
+
+    - name: Fetch ccache
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_path }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: use_ccache and arch == "s390x"
+
     - name: Retag collector builder image to arch specific
       community.docker.docker_image:
         name: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}"

ansible/ci-build-collector.yml

@@ -8,10 +8,14 @@
     COLLECTOR_TAG: "{{ collector_tag }}"
     DISABLE_PROFILING: "{{ disable_profiling }}"
     CMAKE_BUILD_TYPE: "{{ 'Debug' if debug_mode else 'Release' }}"
+    USE_CCACHE: "{{ use_ccache|bool|lower }}"
+    CCACHE_DIR: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}/.ccache"
 
   vars:
     collector_root: "{{ ansible_env.HOME }}/collector"
     local_branch: local
+    ccache_dir: ".ccache"
+    ccache_archive: "docker.tar.gz"
 
   tasks:
     - debug: var=collector_root
@@ -24,9 +28,43 @@
         # than with commit hashes, prevents "reference is not a tree" errors
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
-        recursive: true
+        depth: 1
+        recursive: false
       when: arch == "s390x"
 
+    - name: Clone submodules
+      ansible.builtin.shell: |
+        git submodule update --init --depth 1
+      args:
+        chdir: "{{ collector_root }}"
+      when: arch == "s390x"
+
+    - name: Check if ccache exists
+      delegate_to: localhost
+      ansible.builtin.stat:
+        path: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+      register: ccache_check
+      when: use_ccache and arch == "s390x"
+
+    - name: Create cccache directory
+      ansible.builtin.file:
+        path: "{{ collector_root }}/{{ ccache_dir }}"
+        state: directory
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
+
+    - name: Copy ccache
+      ansible.builtin.copy:
+        src: "{{ github_workspace }}/{{ ccache_dir }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/"
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
+
+    - name: Unarchive ccache
+      ansible.builtin.unarchive:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ collector_root }}/{{ ccache_dir }}"
+        remote_src: true
+      when: use_ccache and arch == "s390x" and ccache_check.stat.exists
+
     - name: Run the builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
@@ -40,6 +78,20 @@
       # ensure this action is printed
       tags: [print_action]
 
+    - name: Create ccache archive
+      ansible.builtin.shell: |
+        rm -f "{{ collector_root }}/{{ ccache_archive }}"
+        cd "{{ collector_root }}/{{ ccache_dir }}"
+        tar czf "{{ collector_root }}/{{ ccache_archive }}" .
+      when: use_ccache and arch == "s390x"
+
+    - name: Fetch ccache
+      ansible.builtin.fetch:
+        src: "{{ collector_root }}/{{ ccache_archive }}"
+        dest: "{{ github_workspace }}/{{ ccache_dir }}/"
+        flat: yes
+      when: use_ccache and arch == "s390x"
+
     - name: Retag collector image to arch specific
       community.docker.docker_image:
         name: "{{ collector_image }}"

ansible/scripts/extract_docker_cache.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# based on https://github.com/reproducible-containers/buildkit-cache-dance
+
+set -e
+
+usage() {
+    echo "Usage: $0 <cache_destination_directory> <target_path_inside_container>"
+    exit 1
+}
+
+if [ "$#" -ne 2 ]; then
+    usage
+fi
+
+CACHE_DEST="$1"
+TARGET_PATH="$2"
+
+CCACHE_ARCHIVE="docker.tar.gz"
+
+SCRATCH_DIR=$(mktemp -d)
+SCRATCH_ARCHIVE=$(mktemp)
+# shellcheck disable=SC2064
+trap "rm -rf ${SCRATCH_DIR}; rm -f ${SCRATCH_ARCHIVE}" EXIT
+
+# Timestamp to bust cache
+date -Iseconds > "${SCRATCH_DIR}/buildstamp"
+
+cat << EOF > "${SCRATCH_DIR}/Dockerfile.extract"
+FROM busybox:1
+COPY buildstamp buildstamp
+RUN --mount=type=cache,target=${TARGET_PATH} \\
+    mkdir -p /var/docker-cache/ \\
+    && cp -p -R ${TARGET_PATH}/. /var/docker-cache/ || true
+EOF
+
+echo "Generated Dockerfile.extract"
+cat "${SCRATCH_DIR}/Dockerfile.extract"
+
+# Build the image and load it into Docker
+docker buildx build -f "${SCRATCH_DIR}/Dockerfile.extract" --tag cache:extract --load "${SCRATCH_DIR}"
+docker images
+
+# Remove any existing cache-container
+docker rm -f cache-container || true
+
+# Create a container from cache:extract
+docker create --name cache-container cache:extract
+docker ps
+
+# Extract the cache from the container
+docker cp -L cache-container:/var/docker-cache - | tar -H posix -x -C "${SCRATCH_DIR}"
+ls "${SCRATCH_DIR}"
+
+# Compress the cache from the container
+(cd "${SCRATCH_DIR}/docker-cache" && chmod -R 777 . && tar czf "${SCRATCH_ARCHIVE}" .)
+
+# Move the cache into its dest
+rm -f "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+mv "${SCRATCH_ARCHIVE}" "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+chmod 666 "${CACHE_DEST}/${CCACHE_ARCHIVE}"
+
+echo "Docker cache extraction completed successfully."