[COMGR][Cache] No-Source Code Cache implementation (llvm#522)

This commit implements a compilation cache for `clang::driver::Command`.

This is a reduced version, that doesn't handle source-code as input,
only the late stages are cached (aiming to ship this quickly with less
verification and negotiations with other teams).

Highlights:

* This cache can be enabled by setting the environment variable AMD_COMGR_CACHE=1
* By default, on linux, the cache is written to $HOME/.cache/comgr.
* The cache takes into account llvm's version and device-libs bitcode: after a change in llvm's HEAD commit new compilations won't hit the cache.
* Cache max size is 5gb and leaving more than 75% of the system's available
space

co-authored by anjenner and jmmartinez

Author: jmmartinez

amd/comgr/CMakeLists.txt

@@ -69,6 +69,9 @@ option(COMGR_BUILD_SHARED_LIBS "Build the shared library"
        ${build_shared_libs_default})
 
 set(SOURCES
+  src/comgr-cache.cpp
+  src/comgr-cache-command.cpp
+  src/comgr-clang-command.cpp
   src/comgr-compiler.cpp
   src/comgr.cpp
   src/comgr-device-libs.cpp

amd/comgr/README.md

@@ -114,6 +114,26 @@ Comgr supports an environment variable to help locate LLVM:
   installation, which is currently used to locate the clang resource directory
   and clang binary path, allowing for additional optimizations.
 
+Comgr utilizes a cache to preserve the results of compilations between executions.
+The cache's status (enabled/disabled), storage location for its results,
+and eviction policy can be manipulated through specific environment variables.
+If an issue arises during cache initialization, the execution will proceed with
+the cache turned off.
+
+By default, the cache is turned off, set the environment variable
+`AMD_COMGR_CACHE=1` to enable it.
+
+* `AMD_COMGR_CACHE`: When unset or set to 0, the cache is turned off.
+* `AMD_COMGR_CACHE_DIR`: If assigned a non-empty value, that value is used as
+  the path for cache storage. If the variable is unset or set to an empty string `""`,
+  it is directed to "$XDG_CACHE_HOME/comgr" (which defaults to
+  "$USER/.cache/comgr" on Linux, and "%LOCALAPPDATA%\cache\comgr"
+  on Microsoft Windows).
+* `AMD_COMGR_CACHE_POLICY`: If assigned a value, the string is interpreted and
+  applied to the cache pruning policy. The cache is pruned only upon program
+  termination. The string format aligns with [Clang's ThinLTO cache pruning policy](https://clang.llvm.org/docs/ThinLTO.html#cache-pruning).
+  The default policy is set as: "prune_interval=1h:prune_expiration=0h:cache_size=75%:cache_size_bytes=30g:cache_size_files=0".
+
 Comgr also supports some environment variables to aid in debugging. These
 include:
 

amd/comgr/cmake/DeviceLibs.cmake

@@ -59,6 +59,7 @@ foreach(AMDGCN_LIB_TARGET ${AMD_DEVICE_LIBS_TARGETS})
   add_dependencies(amd_comgr ${AMDGCN_LIB_TARGET}_header)
 
   list(APPEND TARGETS_INCLUDES "#include \"${header}\"")
+  list(APPEND TARGETS_HEADERS "${INC_DIR}/${header}")
 endforeach()
 
 list(JOIN TARGETS_INCLUDES "\n" TARGETS_INCLUDES)
@@ -110,4 +111,17 @@ list(APPEND TARGETS_DEFS "#undef AMD_DEVICE_LIBS_FUNCTION")
 list(JOIN TARGETS_DEFS "\n" TARGETS_DEFS)
 file(GENERATE OUTPUT ${GEN_LIBRARY_DEFS_INC_FILE} CONTENT "${TARGETS_DEFS}")
 
+# compute the sha256 of the device libraries to detect changes and pass them to comgr (used by the cache)
+find_package(Python3 REQUIRED Interpreter)
+set(DEVICE_LIBS_ID_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/cmake/device-libs-id.py")
+set(DEVICE_LIBS_ID_HEADER ${INC_DIR}/libraries_sha.inc)
+add_custom_command(OUTPUT ${DEVICE_LIBS_ID_HEADER}
+  COMMAND ${Python3_EXECUTABLE} ${DEVICE_LIBS_ID_SCRIPT} --varname DEVICE_LIBS_ID --output ${DEVICE_LIBS_ID_HEADER} ${TARGETS_HEADERS}
+  DEPENDS ${DEVICE_LIBS_ID_SCRIPT} ${TARGETS_HEADERS}
+    COMMENT "Generating ${INC_DIR}/libraries_sha.inc"
+)
+set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${INC_DIR}/libraries_sha.inc)
+add_custom_target(libraries_sha_header DEPENDS ${INC_DIR}/libraries_sha.inc)
+add_dependencies(amd_comgr libraries_sha_header)
+
 include_directories(${INC_DIR})

amd/comgr/cmake/device-libs-id.py

@@ -0,0 +1,20 @@
+from argparse import ArgumentParser
+from hashlib import sha256
+from functools import reduce
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description='Generate id by computing a hash of the generated headers')
+    parser.add_argument("headers", nargs='+', help='List of headers to generate id from')
+    parser.add_argument("--varname", help='Name of the variable to generate', required=True)
+    parser.add_argument("--output", help='Name of the header to generate', required=True)
+
+    args = parser.parse_args()
+    args.headers.sort()
+    
+    hash = sha256()
+    for x in args.headers:
+        hash.update(open(x, 'rb').read())
+    digest_uchar = hash.digest()
+    digest_char = [e if e < 128 else e-256 for e in digest_uchar]
+    digest_elts = ", ".join(map(str, digest_char))
+    print(f"static const char {args.varname}[] = {{{digest_elts}, 0}};", file=open(args.output, 'w'))

amd/comgr/cmake/opencl_pch.cmake

@@ -50,3 +50,7 @@ endfunction()
 
 generate_pch(1.2)
 generate_pch(2.0)
+
+# hash the opencl header and pass the result to comgr compilation
+file(SHA256 ${OPENCL_C_H} OPENCL_C_SHA)
+list(APPEND AMD_COMGR_PRIVATE_COMPILE_DEFINITIONS "OPENCL_C_SHA=${OPENCL_C_SHA}")

amd/comgr/src/comgr-cache-command.cpp

@@ -0,0 +1,101 @@
+#include "comgr-cache-command.h"
+#include "comgr-cache.h"
+#include "comgr-device-libs.h"
+#include "comgr-env.h"
+#include "comgr.h"
+
+#include <clang/Basic/Version.h>
+#include <llvm/ADT/StringExtras.h>
+
+#include <optional>
+
+namespace COMGR {
+using namespace llvm;
+using namespace clang;
+
+namespace {
+// std::isalnum is locale dependent and can have issues
+// depending on the stdlib version and application. We prefer to avoid it
+bool isalnum(char c) {
+  char low[] = {'0', 'a', 'A'};
+  char hi[] = {'9', 'z', 'Z'};
+  for (unsigned i = 0; i != 3; ++i) {
+    if (low[i] <= c && c <= hi[i])
+      return true;
+  }
+  return false;
+}
+
+} // namespace
+
+std::optional<size_t> CachedCommandAdaptor::searchComgrTmpModel(StringRef S) {
+  // Ideally, we would use std::regex_search with the regex
+  // "comgr-[[:alnum:]]{6}". However, due to a bug in stdlibc++
+  // (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85824) we have to roll our
+  // own search of this regular expression. This bug resulted in a crash in
+  // luxmarkv3, during the std::regex constructor.
+  const StringRef Prefix = "comgr-";
+  const size_t AlnumCount = 6;
+
+  size_t N = S.size();
+  size_t Pos = S.find(Prefix);
+
+  size_t AlnumStart = Pos + Prefix.size();
+  size_t AlnumEnd = AlnumStart + AlnumCount;
+  if (Pos == StringRef::npos || N < AlnumEnd)
+    return std::nullopt;
+
+  for (size_t i = AlnumStart; i < AlnumEnd; ++i) {
+    if (!isalnum(S[i]))
+      return std::nullopt;
+  }
+
+  return Pos;
+}
+
+void CachedCommandAdaptor::addString(CachedCommandAdaptor::HashAlgorithm &H,
+                                     StringRef S) {
+  // hash size + contents to avoid collisions
+  // for example, we have to ensure that the result of hashing "AA" "BB" is
+  // different from "A" "ABB"
+  H.update(S.size());
+  H.update(S);
+}
+
+void CachedCommandAdaptor::addFileContents(
+    CachedCommandAdaptor::HashAlgorithm &H, StringRef Buf) {
+  // this is a workaround temporary paths getting in the output files of the
+  // different commands in #line directives in preprocessed files, and the
+  // ModuleID or source_filename in the bitcode.
+  while (!Buf.empty()) {
+    std::optional<size_t> ComgrTmpPos = searchComgrTmpModel(Buf);
+    if (!ComgrTmpPos) {
+      addString(H, Buf);
+      break;
+    }
+
+    StringRef ToHash = Buf.substr(0, *ComgrTmpPos);
+    addString(H, ToHash);
+    Buf = Buf.substr(ToHash.size() + StringRef("comgr-xxxxxx").size());
+  }
+}
+
+Expected<CachedCommandAdaptor::Identifier>
+CachedCommandAdaptor::getIdentifier() const {
+  CachedCommandAdaptor::HashAlgorithm H;
+  H.update(getClass());
+  H.update(env::shouldEmitVerboseLogs());
+  addString(H, getClangFullVersion());
+  addString(H, getComgrHashIdentifier());
+  addString(H, getDeviceLibrariesIdentifier());
+
+  if (Error E = addInputIdentifier(H))
+    return E;
+
+  addOptionsIdentifier(H);
+
+  CachedCommandAdaptor::Identifier Id;
+  toHex(H.final(), true, Id);
+  return Id;
+}
+} // namespace COMGR

amd/comgr/src/comgr-cache-command.h

@@ -0,0 +1,78 @@
+/*******************************************************************************
+ *
+ * University of Illinois/NCSA
+ * Open Source License
+ *
+ * Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * with the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ *     * Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimers.
+ *
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimers in the
+ *       documentation and/or other materials provided with the distribution.
+ *
+ *     * Neither the names of Advanced Micro Devices, Inc. nor the names of its
+ *       contributors may be used to endorse or promote products derived from
+ *       this Software without specific prior written permission.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
+ * THE SOFTWARE.
+ *
+ ******************************************************************************/
+
+#ifndef COMGR_CACHE_COMMAND_H
+#define COMGR_CACHE_COMMAND_H
+
+#include "amd_comgr.h"
+
+#include <clang/Driver/Action.h>
+#include <llvm/Support/Error.h>
+#include <llvm/Support/MemoryBuffer.h>
+#include <llvm/Support/SHA256.h>
+
+namespace llvm {
+class raw_ostream;
+}
+
+namespace COMGR {
+class CachedCommandAdaptor {
+public:
+  using ActionClass = clang::driver::Action::ActionClass;
+  using HashAlgorithm = llvm::SHA256;
+  using Identifier = llvm::SmallString<64>;
+
+  llvm::Expected<Identifier> getIdentifier() const;
+
+  virtual bool canCache() const = 0;
+  virtual llvm::Error writeExecuteOutput(llvm::StringRef CachedBuffer) = 0;
+  virtual llvm::Expected<llvm::StringRef> readExecuteOutput() = 0;
+  virtual amd_comgr_status_t execute(llvm::raw_ostream &LogS) = 0;
+
+  virtual ~CachedCommandAdaptor() = default;
+
+  // helper to work around the comgr-xxxxx string appearing in files
+  static void addFileContents(HashAlgorithm &H, llvm::StringRef Buf);
+  static void addString(HashAlgorithm &H, llvm::StringRef S);
+  static std::optional<size_t> searchComgrTmpModel(llvm::StringRef S);
+
+protected:
+  virtual ActionClass getClass() const = 0;
+  virtual void addOptionsIdentifier(HashAlgorithm &) const = 0;
+  virtual llvm::Error addInputIdentifier(HashAlgorithm &) const = 0;
+};
+} // namespace COMGR
+
+#endif