From e52a0659909973a8a9217fe5de5b16bd07bdbc74 Mon Sep 17 00:00:00 2001 From: Jacob Maine Date: Wed, 28 Apr 2021 17:19:28 -0700 Subject: [PATCH 1/3] Cache clojure dependencies This speeds up builds of Clojure repos by avoiding network calls to download dependencies. When a repo has a `deps.edn` file, we assume it is a Clojure repo and would benefit from pre-populating and caching its primary dependencies. Since Clojure dependencies are ultimately stored in the global `.m2` directory, along with other Java dependencies, it suffices to restore `.m2`. The directory is automatically populated during a build performed with the [Clojure CLI tools][clojure-cli]. This extends 52f8eb9, which added support for repos which are built with the Clojure CLI tools. It is similar to f3a4d49 which introduced caching for Leiningen and Boot, two other ways of building Clojure code. [clojure-cli]: https://clojure.org/reference/deps_and_cli --- run-build-functions.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run-build-functions.sh b/run-build-functions.sh index 328deba3..5a3ec41b 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -581,6 +581,12 @@ install_dependencies() { fi fi + # Clojure CLI + if [ -f deps.edn ] + then + restore_home_cache ".m2" "maven dependencies" + fi + # Hugo if [ -n "$HUGO_VERSION" ] then From 7bcb781d73bd79ce50e420c928edbff13f96428b Mon Sep 17 00:00:00 2001 From: Jacob Maine Date: Wed, 28 Apr 2021 17:35:13 -0700 Subject: [PATCH 2/3] Cache Clojure classpath and prefill caches For repos that are built with the [Clojure CLI tools][clojure-cli] (i.e., repos marked with a `deps.edn` file), we cache and restore the `.cpcache` directory. Clojure uses this directory to avoid unnecessarily re-calculating the JVM classpath. For most projects this additional cache will not lead to a large improvement in build times. We also download dependencies to the `.m2` cache before handing off to the build script. Technically, this is unnecessary since, presumably, the build script uses the Clojure CLI tools which have a side-effect of populating `.m2`. Therefore, the benefit of this change is simply to separate the two steps, isolating any errors that occure while fetching dependencies. [clojure-cli]: https://clojure.org/reference/deps_and_cli --- Dockerfile | 2 ++ run-build-functions.sh | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Dockerfile b/Dockerfile index e685c0c0..e2fa02e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -350,6 +350,8 @@ RUN mkdir /opt/boot-clj && cd /opt/boot-clj && \ chmod +x boot && \ ln -s /opt/boot-clj/boot /usr/local/bin/boot +# TODO: When upgrading to >= 1.10.1.672, also update approach for populating .m2 with clojure deps: +# In run-build-functions.sh replace `clojure -Spath -Sforce >/dev/null` with more modern `clojure -P` RUN curl -sL https://download.clojure.org/install/linux-install-1.10.1.492.sh | bash USER buildbot diff --git a/run-build-functions.sh b/run-build-functions.sh index 5a3ec41b..23400226 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -585,6 +585,15 @@ install_dependencies() { if [ -f deps.edn ] then restore_home_cache ".m2" "maven dependencies" + restore_cwd_cache ".cpcache" "clojure classpath" + echo "Installing Clojure dependencies" + if clojure -Spath -Sforce >/dev/null + then + echo "Clojure dependencies installed" + else + echo "Error during Clojure install" + exit 1 + fi fi # Hugo @@ -706,6 +715,7 @@ cache_artifacts() { cache_cwd_directory ".venv" "python virtualenv" cache_cwd_directory ".build" "swift build" + cache_cwd_directory ".cpcache" "clojure classpath" cache_cwd_directory ".netlify/plugins" "build plugins" cache_cwd_directory ".netlify/rust-functions-cache" "Rust functions" From f2d2b9c5b5dd69c09ba2ab7592d8a95217cf2b89 Mon Sep 17 00:00:00 2001 From: Jacob Maine Date: Thu, 29 Apr 2021 09:40:26 -0700 Subject: [PATCH 3/3] Cache Clojure git dependencies The [Clojure CLI tools][clojure-cli] have a mechanism of downloading dependencies directly from git repositories rather than from a package manager. This patch caches the directory which stores these downloads, speeding up builds of repos that use these types of dependencies. [clojure-cli]: https://clojure.org/guides/deps_and_cli --- run-build-functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run-build-functions.sh b/run-build-functions.sh index 23400226..7648b3a8 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -585,6 +585,7 @@ install_dependencies() { if [ -f deps.edn ] then restore_home_cache ".m2" "maven dependencies" + restore_home_cache ".gitlibs" "clojure git dependencies" restore_cwd_cache ".cpcache" "clojure classpath" echo "Installing Clojure dependencies" if clojure -Spath -Sforce >/dev/null @@ -730,6 +731,7 @@ cache_artifacts() { cache_home_directory ".emacs.d" "emacs cache" cache_home_directory ".m2" "maven dependencies" cache_home_directory ".boot" "boot dependencies" + cache_home_directory ".gitlibs" "clojure git dependencies" cache_home_directory ".composer" "composer dependencies" cache_home_directory ".homebrew-cache", "homebrew cache" cache_home_directory ".rustup" "rust rustup cache"