Enzyme Jacobian for PhasorDynamics::Load within GridKit

CMakeLists.txt

@@ -103,6 +103,8 @@ endif()
 
 if(${GRIDKIT_ENABLE_ENZYME})
   include(FindEnzyme)
+  # todo Add a centralized configuration file
+  add_definitions(-DGRIDKIT_ENABLE_ENZYME)
 endif()
 
 # Macro that adds libraries

cmake/EnzymeAddLibrary.cmake

@@ -0,0 +1,123 @@
+# 
+#[[
+
+Macro to manually compile with Enzyme
+
+Author(s):
+- Asher Mancinelli <ashermancinelli@gmail.com>
+- Nicholson Koukpaizan <koukpaizannk@ornl.gov>
+
+]]
+
+macro(enzyme_build_object)
+  set(options)
+  set(oneValueArgs NAME)
+  set(multiValueArgs SOURCES LINK_LIBRARIES INCLUDE_DIRECTORIES)
+  cmake_parse_arguments(enzyme_build_object "${options}" "${oneValueArgs}"
+    "${multiValueArgs}" ${ARGN})
+
+  set(PHASE2 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_build_object_NAME}.bc")
+  set(PHASE3 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_build_object_NAME}_enzyme.ll")
+  set(PHASE4 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_build_object_NAME}_opt.ll")
+  set(PHASE5 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_build_object_NAME}")
+
+  set(OBJS "")
+  set(includes "${enzyme_build_object_INCLUDE_DIRECTORIES}")
+
+  foreach(lib ${enzyme_build_object_LINK_LIBRARIES})
+    get_target_property(include ${lib} INCLUDE_DIRECTORIES)
+    set(includes "${includes}" ${include})
+
+    get_target_property(libsource ${lib} SOURCES)
+    string(FIND "${libsource}" "TARGET" found)
+    if(NOT(${found} EQUAL -1))
+      list(APPEND LINKER_FLAGS "-Wl,${libsource}")
+    endif()
+  endforeach()
+
+  foreach(dir ${includes})
+    if(EXISTS ${dir})
+      list(APPEND INCLUDE_COMPILER_LIST "-I${dir}")
+    endif()
+  endforeach()
+
+  foreach(SRC ${enzyme_build_object_SOURCES})
+    set(PHASE0 "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}")
+    set(PHASE1 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_build_object_NAME}_${SRC}_compile.o")
+    add_custom_command(
+      DEPENDS ${PHASE0} 
+      OUTPUT ${PHASE1}
+      COMMAND ${CMAKE_CXX_COMPILER} -flto -c ${PHASE0} ${INCLUDE_COMPILER_LIST} -O2 -fno-vectorize -ffast-math -fno-unroll-loops -fpass-plugin=${ENZYME_CLANG_PLUGIN_LIBRARY} -Xclang -load -Xclang ${ENZYME_CLANG_PLUGIN_LIBRARY} -mllvm -enable-load-pre=0 -mllvm -enzyme-auto-sparsity=1 -o ${PHASE1}
+      COMMENT "Compiling ${SRC} to object file for target ${enzyme_build_object_NAME}"
+      )
+    set(OBJS "${OBJS} ${PHASE1}")
+  endforeach()
+
+  cmake_language(EVAL CODE "
+  add_custom_command(
+    DEPENDS ${OBJS}
+    OUTPUT ${PHASE2}
+    COMMAND ${GRIDKIT_LLVM_LINK} ${OBJS} -o ${PHASE2}
+    COMMENT \"Linking object files to LLVM bytecode for target ${enzyme_build_object_NAME}\"
+    )
+  ")
+
+  add_custom_command(
+    DEPENDS ${PHASE2}
+    OUTPUT ${PHASE3}
+    COMMAND ${GRIDKIT_OPT} ${PHASE2} -load-pass-plugin=${ENZYME_LLVM_PLUGIN_LIBRARY} -passes=enzyme -o ${PHASE3} -S
+    COMMENT "Running Enzyme opt pass on target ${enzyme_build_object_NAME}"
+    )
+
+  add_custom_command(
+    DEPENDS ${PHASE3}
+    OUTPUT ${PHASE4}
+    COMMAND ${GRIDKIT_OPT} ${PHASE3} -O2 -o ${PHASE4} -S
+    COMMENT "Running remaining opt passes on target ${enzyme_build_object_NAME}"
+    )
+
+  add_custom_command(
+    DEPENDS ${PHASE4}
+    OUTPUT ${PHASE5}
+    COMMAND ${CMAKE_CXX_COMPILER} -c ${PHASE4} -o ${PHASE5}
+    COMMENT "Generating optimized object file for target ${enzyme_build_object_NAME}"
+    )
+endmacro()
+
+macro(enzyme_add_executable)
+  set(options)
+  set(oneValueArgs NAME)
+  set(multiValueArgs SOURCES LINK_LIBRARIES INCLUDE_DIRECTORIES)
+  cmake_parse_arguments(enzyme_add_executable "${options}" "${oneValueArgs}"
+    "${multiValueArgs}" ${ARGN})
+
+  enzyme_build_object(
+    NAME "${enzyme_add_executable_NAME}.o"
+    SOURCES ${enzyme_add_executable_SOURCES}
+    LINK_LIBRARIES ${enzyme_add_executable_LINK_LIBRARIES}
+    INCLUDE_DIRECTORIES ${enzyme_add_executable_INCLUDE_DIRECTORIES}
+  )
+
+  add_executable("${enzyme_add_executable_NAME}" "${enzyme_add_executable_NAME}.o")
+  set_target_properties("${enzyme_add_executable_NAME}" PROPERTIES LINKER_LANGUAGE CXX)
+  target_link_libraries("${enzyme_add_executable_NAME}" ${enzyme_add_executable_LINK_LIBRARIES})
+endmacro()
+
+macro(enzyme_add_library)
+  set(options)
+  set(oneValueArgs NAME)
+  set(multiValueArgs SOURCES LINK_LIBRARIES INCLUDE_DIRECTORIES)
+  cmake_parse_arguments(enzyme_add_library "${options}" "${oneValueArgs}"
+    "${multiValueArgs}" ${ARGN})
+
+  enzyme_build_object(
+    NAME "${enzyme_add_library_NAME}.o"
+    SOURCES ${enzyme_add_library_SOURCES}
+    LINK_LIBRARIES ${enzyme_add_library_LINK_LIBRARIES}
+    INCLUDE_DIRECTORIES ${enzyme_add_library_INCLUDE_DIRECTORIES}
+  )
+
+  add_library("${enzyme_add_library_NAME}" "${enzyme_add_library_NAME}.o")
+  set_target_properties("${enzyme_add_library_NAME}" PROPERTIES LINKER_LANGUAGE CXX)
+  target_link_libraries("${enzyme_add_library_NAME}" ${enzyme_add_library_LINK_LIBRARIES})
+endmacro()

cmake/FindEnzyme.cmake

@@ -59,82 +59,3 @@ find_program(GRIDKIT_OPT opt
              bin
              REQUIRED)
 message(STATUS "opt: ${GRIDKIT_OPT}")
-
-macro(enzyme_add_executable)
-  set(options)
-  set(oneValueArgs NAME)
-  set(multiValueArgs SOURCES LINK_LIBRARIES INCLUDE_DIRECTORIES)
-  cmake_parse_arguments(enzyme_add_executable "${options}" "${oneValueArgs}"
-    "${multiValueArgs}" ${ARGN})
-
-  set(PHASE2 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}.bc")
-  set(PHASE3 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_enzyme.ll")
-  set(PHASE4 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_opt.ll")
-  set(PHASE5 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}")
-
-  set(OBJS "")
-  set(includes "${enzyme_add_executable_INCLUDE_DIRECTORIES}")
-
-  foreach(lib ${enzyme_add_executable_LINK_LIBRARIES})
-    get_target_property(include ${lib} INCLUDE_DIRECTORIES)
-    set(includes "${includes}" ${include})
-
-    get_target_property(libsource ${lib} SOURCES)
-    string(FIND "${libsource}" "TARGET" found)
-    if(NOT(${found} EQUAL -1))
-      list(APPEND LINKER_FLAGS "-Wl,${libsource}")
-    endif()
-  endforeach()
-
-  foreach(dir ${includes})
-    if(EXISTS ${dir})
-      list(APPEND INCLUDE_COMPILER_LIST "-I${dir}")
-    endif()
-  endforeach()
-
-  foreach(SRC ${enzyme_add_executable_SOURCES})
-    set(PHASE0 "${CMAKE_CURRENT_SOURCE_DIR}/${SRC}")
-    set(PHASE1 "${CMAKE_CURRENT_BINARY_DIR}/${enzyme_add_executable_NAME}_${SRC}_compile.o")
-    add_custom_command(
-      DEPENDS ${PHASE0} 
-      OUTPUT ${PHASE1}
-      COMMAND ${CMAKE_CXX_COMPILER} -flto -c ${PHASE0} ${INCLUDE_COMPILER_LIST} -O2 -fno-vectorize -ffast-math -fno-unroll-loops -fpass-plugin=${ENZYME_CLANG_PLUGIN_LIBRARY} -Xclang -load -Xclang ${ENZYME_CLANG_PLUGIN_LIBRARY} -mllvm -enable-load-pre=0 -mllvm -enzyme-auto-sparsity=1 -o ${PHASE1}
-      COMMENT "Compiling ${SRC} to object file for target ${enzyme_add_executable_NAME}"
-      )
-    set(OBJS "${OBJS} ${PHASE1}")
-  endforeach()
-
-  cmake_language(EVAL CODE "
-  add_custom_command(
-    DEPENDS ${OBJS}
-    OUTPUT ${PHASE2}
-    COMMAND ${GRIDKIT_LLVM_LINK} ${OBJS} -o ${PHASE2}
-    COMMENT \"Linking object files to LLVM bytecode for target ${enzyme_add_executable_NAME}\"
-    )
-  ")
-
-  add_custom_command(
-    DEPENDS ${PHASE2}
-    OUTPUT ${PHASE3}
-    COMMAND ${GRIDKIT_OPT} ${PHASE2} -load-pass-plugin=${ENZYME_LLVM_PLUGIN_LIBRARY} -passes=enzyme -o ${PHASE3} -S
-    COMMENT "Running Enzyme opt pass on target ${enzyme_add_executable_NAME}"
-    )
-
-  add_custom_command(
-    DEPENDS ${PHASE3}
-    OUTPUT ${PHASE4}
-    COMMAND ${GRIDKIT_OPT} ${PHASE3} -O2 -o ${PHASE4} -S
-    COMMENT "Running remaining opt passes on target ${enzyme_add_executable_NAME}"
-    )
-
-  add_custom_command(
-    DEPENDS ${PHASE4} ${enzyme_add_executable_LINK_LIBRARIES}
-    OUTPUT ${PHASE5}
-    COMMAND ${CMAKE_CXX_COMPILER} ${LINKER_FLAGS} ${PHASE4} -o ${PHASE5}
-    )
-
-  add_custom_target(
-    "${enzyme_add_executable_NAME}_target" ALL
-    DEPENDS ${PHASE5}
-    )
-endmacro()

examples/Enzyme/Library/Scalar/CMakeLists.txt

@@ -1,6 +1,4 @@
-enzyme_add_executable(
-  NAME EnzymeLibScalarCheck 
-  SOURCES EnzymeScalar.cpp ScalarModel.cpp
-)
+add_executable(EnzymeLibScalarCheck EnzymeScalar.cpp ScalarModel.cpp)
+target_link_libraries(EnzymeLibScalarCheck ClangEnzymeFlags GRIDKIT::Utilities)
 
 add_test(NAME "EnzymeLibScalarCheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/EnzymeLibScalarCheck)

examples/Enzyme/Library/Scalar/EnzymeScalar.cpp

@@ -1,7 +1,7 @@
 #include <iostream>
-#include <limits>
 
 #include "ScalarModel.hpp"
+#include <Utilities/Testing.hpp>
 
 /**
  * @brief Example that computes the derivative of a library function
@@ -23,7 +23,7 @@ int main()
   double dsq = scalar_model.getDerivativeValue();
 
   std::cout << "x = " << var << ", x^2 = " << sq << ", d(x^2)/dx = " << dsq << "\n";
-  if (std::abs(dsq - 2.0 * var) > std::numeric_limits<double>::epsilon())
+  if (!GridKit::Testing::isEqual(dsq, 2.0 * var))
   {
     fail++;
     std::cout << "Result incorrect\n";

examples/Enzyme/Library/Vector/CMakeLists.txt

@@ -1,7 +1,4 @@
-enzyme_add_executable(
-  NAME EnzymeLibVectorCheck 
-  SOURCES EnzymeVector.cpp VectorModel.cpp
-  LINK_LIBRARIES GRIDKIT::DenseMatrix
-)
+add_executable(EnzymeLibVectorCheck EnzymeVector.cpp VectorModel.cpp)
+target_link_libraries(EnzymeLibVectorCheck ClangEnzymeFlags GRIDKIT::DenseMatrix GRIDKIT::Utilities)
 
 add_test(NAME "EnzymeLibVectorCheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/EnzymeLibVectorCheck)

examples/Enzyme/Library/Vector/EnzymeVector.cpp

@@ -1,7 +1,7 @@
 #include <iostream>
-#include <limits>
 
 #include "VectorModel.hpp"
+#include <Utilities/Testing.hpp>
 
 /**
  * @brief Example that computes the Jacobian of a vector-valued residual
@@ -20,12 +20,12 @@ inline double dsquare_ref_scalar(double x)
 DenseMatrix dsquare_ref(std::vector<double> x, std::vector<double> y)
 {
   DenseMatrix jac(x.size(), y.size());
-  for (int idy = 0; idy < y.size(); ++idy)
+  for (size_t idy = 0; idy < y.size(); ++idy)
   {
-    for (int idx = 0; idx < x.size(); ++idx)
+    for (size_t idx = 0; idx < x.size(); ++idx)
     {
-      if (idx == idy)
-        jac.setValue(idx, idy, dsquare_ref_scalar(x[idx]));
+      if (idy <= idx)
+        jac.setValue(idx, idy, dsquare_ref_scalar(x[idy]));
     }
   }
   return jac;
@@ -34,12 +34,12 @@ DenseMatrix dsquare_ref(std::vector<double> x, std::vector<double> y)
 int main()
 {
   // Size and variable declarations
-  constexpr int       n = 10;
+  constexpr size_t    n = 10;
   std::vector<double> var(n);
 
   // Random input values
-  srand(time(NULL));
-  for (int idx = 0; idx < var.size(); ++idx)
+  srand(static_cast<unsigned int>(time(NULL)));
+  for (size_t idx = 0; idx < var.size(); ++idx)
   {
     var[idx] = rand();
   }
@@ -59,11 +59,11 @@ int main()
   // Check
   int  fail    = 0;
   bool verbose = true;
-  for (int idy = 0; idy < res.size(); ++idy)
+  for (size_t idy = 0; idy < res.size(); ++idy)
   {
-    for (int idx = 0; idx < var.size(); ++idx)
+    for (size_t idx = 0; idx < var.size(); ++idx)
     {
-      if (std::abs(jac.getValue(idx, idy) - jac_ref.getValue(idx, idy)) > std::numeric_limits<double>::epsilon())
+      if (!GridKit::Testing::isEqual(jac.getValue(idx, idy), jac_ref.getValue(idx, idy)))
       {
         fail++;
         if (verbose)

examples/Enzyme/Library/Vector/VectorModel.cpp

@@ -4,7 +4,7 @@
 
 #include "EnzymeWrapper.hpp"
 
-VectorModel::VectorModel(int n)
+VectorModel::VectorModel(size_t n)
   : x_(n),
     f_(n),
     df_dx_(n, n)
@@ -18,15 +18,19 @@ inline double VectorModel::square_scalar(double x)
 
 void VectorModel::square(std::vector<double>& x, std::vector<double>& y)
 {
-  for (int idx = 0; idx < x.size(); ++idx)
+  for (size_t idx = 0; idx < x.size(); ++idx)
   {
-    y[idx] = this->square_scalar(x[idx]);
+    y[idx] = 0.0;
+    for (size_t idy = 0; idy <= idx; idy++)
+    {
+      y[idx] += this->square_scalar(x[idy]);
+    }
   }
 }
 
 void VectorModel::setVariable(std::vector<double> x)
 {
-  for (int idx = 0; idx < x.size(); ++idx)
+  for (size_t idx = 0; idx < x.size(); ++idx)
   {
     x_[idx] = x[idx];
   }
@@ -39,13 +43,13 @@ void VectorModel::evalResidual()
 
 void VectorModel::evalJacobian()
 {
-  const int           n = x_.size();
+  const size_t        n = x_.size();
   std::vector<double> v(n);
   VectorModel         d_vector_model(n);
-  for (int idy = 0; idy < n; ++idy)
+  for (size_t idy = 0; idy < n; ++idy)
   {
     // Elementary vector for Jacobian-vector product
-    for (int idx = 0; idx < n; ++idx)
+    for (size_t idx = 0; idx < n; ++idx)
     {
       v[idx] = 0.0;
     }
@@ -60,7 +64,7 @@ void VectorModel::evalJacobian()
         &d_vector_model);
 
     // Store result
-    for (int idx = 0; idx < n; ++idx)
+    for (size_t idx = 0; idx < n; ++idx)
     {
       df_dx_.setValue(idx, idy, d_res[idx]);
     }

examples/Enzyme/Library/Vector/VectorModel.hpp

@@ -17,7 +17,7 @@ class VectorModel
   void                square(std::vector<double>&, std::vector<double>&);
 
 public:
-  VectorModel(int);
+  VectorModel(size_t);
   void                 setVariable(std::vector<double>);
   void                 evalResidual();
   void                 evalJacobian();

examples/Enzyme/PowerElectronics/CMakeLists.txt

@@ -1,7 +1,5 @@
-enzyme_add_executable(
-  NAME EnzymePowerElectronicsCheck 
-  SOURCES main.cpp
-  LINK_LIBRARIES GRIDKIT::DenseMatrix GRIDKIT::power_elec_disgen
-)
+add_executable(EnzymePowerElectronicsCheck main.cpp)
+target_compile_options(EnzymePowerElectronicsCheck PUBLIC -fno-vectorize -ffast-math -fno-unroll-loops)
+target_link_libraries(EnzymePowerElectronicsCheck ClangEnzymeFlags GRIDKIT::DenseMatrix GRIDKIT::power_elec_disgen GRIDKIT::Utilities)
 
 add_test(NAME "EnzymePowerElectronicsCheck" COMMAND ${CMAKE_CURRENT_BINARY_DIR}/EnzymePowerElectronicsCheck)