Install and compile with ccache for faster builds

Author: prigoyal

docker/common/install_base.sh

@@ -13,6 +13,7 @@ apt-get install -y --no-install-recommends \
   wget \
   unzip \
   vim \
+  autoconf \
   automake \
   libtool \
   valgrind \
@@ -33,12 +34,48 @@ apt-get install -y --no-install-recommends libcilkrts5 gcc-$GCC_VERSION g++-$GCC
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GCC_VERSION 50
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$GCC_VERSION 50
 
-export CC=/usr/bin/gcc
-export CXX=/usr/bin/g++
+# Install ccache from source. Needs 3.4 or later for ccbin support
+# Needs specific branch to work with nvcc (ccache/ccache#145)
+# Also pulls in a commit that disables documentation generation,
+# as this requires asciidoc to be installed (which pulls in a LOT of deps).
+echo "Installing ccache"
+pushd /tmp
+git clone https://github.com/pietern/ccache -b ccbin
+pushd ccache
+./autogen.sh
+./configure --prefix=/usr/local
+make "-j$(nproc)" install
+popd
+popd
+
+# Install ccache symlink wrappers
+# A good read on ccache: https://software.intel.com/en-us/articles/accelerating-compilation-part-1-ccache
+echo "Setting up ccache wrappers"
+pushd /usr/local/bin
+ln -sf "$(which ccache)" cc
+ln -sf "$(which ccache)" c++
+ln -sf "$(which ccache)" gcc
+ln -sf "$(which ccache)" g++
+ln -sf "$(which ccache)" x86_64-linux-gnu-gcc
+# Install ccache wrapper for nvcc. We are using NVIDIA image so nvcc is there.
+ln -sf "$(which ccache)" nvcc
+# set the cache limit
+ccache -M 25Gi
+
+export CCACHE_WRAPPER_DIR="$PWD/ccache"
+export PATH="$CCACHE_WRAPPER_DIR:$PATH"
+# CMake should use ccache symlink for nvcc
+export CUDA_NVCC_EXECUTABLE="$PWD/nvcc"
+popd
+
+echo "Setting CC and CXX env variables"
+export CC=$(which gcc)
+export CXX=$(which g++)
+echo "CC: ${CC}"
+echo "CXX: ${CXX}"
 
 # install cmake - this unifies trusty/xenial cmake version > 3.4.3
 [ -n "$CMAKE_VERSION" ]
-
 # Turn 3.6.3 into v3.6
 path=$(echo "${CMAKE_VERSION}" | sed -e 's/\([0-9].[0-9]\+\).*/v\1/')
 file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
@@ -49,7 +86,6 @@ tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
 rm -f cmake-*.tar.gz
 popd
 
-
 # LLVM+Clang-Tapir5.0
 export LLVM_SOURCES=/tmp/llvm_sources-tapir5.0
 mkdir -p $LLVM_SOURCES

docker/docker_build.sh

@@ -54,7 +54,6 @@ echo "============Summary Ended================"
 docker build \
        --build-arg "BUILD_ENVIRONMENT=${image}" \
        --build-arg "CMAKE_VERSION=${CMAKE_VERSION}" \
-       --build-arg "EC2=${EC2:-}" \
        --build-arg "JENKINS=${JENKINS:-}" \
        --build-arg "JENKINS_UID=${JENKINS_UID:-}" \
        --build-arg "JENKINS_GID=${JENKINS_GID:-}" \

docker/ubuntu-cuda-conda/Dockerfile

@@ -22,6 +22,9 @@ ARG GCC_VERSION
 ARG CMAKE_VERSION
 ADD ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
+ENV PATH /usr/local/bin/ccache:$PATH
+ENV CUDA_NVCC_EXECUTABLE /usr/local/bin/nvcc
+ENV PATH /usr/local/cuda/bin:$PATH
 
 # Anaconda3
 ADD ./common/install_conda.sh install_conda.sh
@@ -45,8 +48,8 @@ ARG BUILD_ENVIRONMENT
 ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 
 # Set all the environment variables now
-ENV CC /usr/bin/gcc
-ENV CXX /usr/bin/g++
+ENV CC /usr/local/bin/gcc
+ENV CXX /usr/local/bin/g++
 ENV PATH /opt/conda/bin:$PATH
 ENV CMAKE_VERSION cmake
 

docker/ubuntu-cuda/Dockerfile

@@ -17,6 +17,9 @@ ARG GCC_VERSION
 ARG CMAKE_VERSION
 ADD ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
+ENV PATH /usr/local/bin/ccache:$PATH
+ENV CUDA_NVCC_EXECUTABLE /usr/local/bin/nvcc
+ENV PATH /usr/local/cuda/bin:$PATH
 
 # Python3
 ADD ./common/install_pip_python.sh install_pip_python.sh
@@ -39,8 +42,8 @@ ARG BUILD_ENVIRONMENT
 ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 
 # Set all the environment variables now
-ENV CC /usr/bin/gcc
-ENV CXX /usr/bin/g++
+ENV CC /usr/local/bin/gcc
+ENV CXX /usr/local/bin/g++
 ENV PATH /usr/bin/python3:$PATH
 ENV CMAKE_VERSION cmake