create and validate build_variables.bzl

CMakeLists.txt

@@ -44,6 +44,7 @@
 
 cmake_minimum_required(VERSION 3.19)
 project(executorch)
+include(build/Codegen.cmake)
 include(build/Utils.cmake)
 include(CMakeDependentOption)
 
@@ -401,6 +402,7 @@ if(NOT EXECUTORCH_SRCS_FILE)
   message(STATUS "executorch: Generating source lists")
   set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake")
   extract_sources(${EXECUTORCH_SRCS_FILE})
+  validate_build_variables()
 endif()
 
 # This file defines the `_<target>__srcs` variables used below.

build/Codegen.cmake

@@ -9,7 +9,7 @@
 
 # Selective build. See codegen/tools/gen_oplist.py for how to use these
 # arguments.
-include(${EXECUTORCH_ROOT}/build/Utils.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/Utils.cmake)
 
 function(gen_selected_ops)
   set(arg_names LIB_NAME OPS_SCHEMA_YAML ROOT_OPS INCLUDE_ALL_OPS)
@@ -97,7 +97,7 @@ function(generate_bindings_for_kernels)
       --tags-path=${site-packages-out}/torchgen/packaged/ATen/native/tags.yaml
       --aten-yaml-path=${site-packages-out}/torchgen/packaged/ATen/native/native_functions.yaml
       --op-selection-yaml-path=${_oplist_yaml}
-    )
+  )
   if(GEN_ADD_EXCEPTION_BOUNDARY)
     set(_gen_command "${_gen_command}" --add-exception-boundary)
   endif()
@@ -217,3 +217,127 @@ function(merge_yaml)
     WORKING_DIRECTORY ${EXECUTORCH_ROOT}
   )
 endfunction()
+
+# Append the file list in the variable named `name` in build/build_variables.bzl
+# to the variable named `outputvar` in the caller's scope.
+function(append_filelist name outputvar)
+  # configure_file adds its input to the list of CMAKE_RERUN dependencies
+  configure_file(
+    ${PROJECT_SOURCE_DIR}/build/build_variables.bzl
+    ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY
+  )
+  execute_process(
+    COMMAND
+      "${PYTHON_EXECUTABLE}" -c
+      "exec(open('${PROJECT_SOURCE_DIR}/build/build_variables.bzl').read());print(';'.join(${name}))"
+    WORKING_DIRECTORY "${_rootdir}"
+    RESULT_VARIABLE _retval
+    OUTPUT_VARIABLE _tempvar
+    ERROR_VARIABLE _stderr
+  )
+  if(NOT _retval EQUAL 0)
+    message(
+      FATAL_ERROR
+        "Failed to fetch filelist ${name} from build_variables.bzl with output ${_tempvar} and stderr ${_stderr}"
+    )
+  endif()
+  string(REPLACE "\n" "" _tempvar "${_tempvar}")
+  list(APPEND ${outputvar} ${_tempvar})
+  set(${outputvar}
+      "${${outputvar}}"
+      PARENT_SCOPE
+  )
+endfunction()
+
+# Fail the build if the src lists in build_variables.bzl do not match the src
+# lists extracted from Buck and placed into EXECUTORCH_SRCS_FILE. This is
+# intended to be a safety mechanism while we are in the process of removing Buck
+# from the CMake build and replacing it with build_variables.bzl; if you are
+# seeing failures after you have intentionally changed Buck srcs, then simply
+# update build_variables.bzl. If you are seeing failures after changing
+# something about the build system, make sure your changes will work both before
+# and after we finish replacing Buck with build_variables.bzl, which should
+# involve getting these lists to match!
+function(validate_build_variables)
+  include(${EXECUTORCH_SRCS_FILE})
+  set(BUILD_VARIABLES_FILELISTS
+      EXECUTORCH_SRCS
+      EXECUTORCH_CORE_SRCS
+      PORTABLE_KERNELS_SRCS
+      OPTIMIZED_KERNELS_SRCS
+      QUANTIZED_KERNELS_SRCS
+      PROGRAM_SCHEMA_SRCS
+      OPTIMIZED_CPUBLAS_SRCS
+      OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS
+      EXTENSION_DATA_LOADER_SRCS
+      EXTENSION_MODULE_SRCS
+      EXTENSION_RUNNER_UTIL_SRCS
+      EXTENSION_LLM_RUNNER_SRCS
+      EXTENSION_TENSOR_SRCS
+      EXTENSION_THREADPOOL_SRCS
+      EXTENSION_TRAINING_SRCS
+      TRAIN_XOR_SRCS
+      EXECUTOR_RUNNER_SRCS
+      SIZE_TEST_SRCS
+      MPS_EXECUTOR_RUNNER_SRCS
+      MPS_BACKEND_SRCS
+      MPS_SCHEMA_SRCS
+      XNN_EXECUTOR_RUNNER_SRCS
+      XNNPACK_BACKEND_SRCS
+      XNNPACK_SCHEMA_SRCS
+      VULKAN_SCHEMA_SRCS
+      CUSTOM_OPS_SRCS
+      LLAMA_RUNNER_SRCS
+  )
+  set(BUILD_VARIABLES_VARNAMES
+      _executorch__srcs
+      _executorch_core__srcs
+      _portable_kernels__srcs
+      _optimized_kernels__srcs
+      _quantized_kernels__srcs
+      _program_schema__srcs
+      _optimized_cpublas__srcs
+      _optimized_native_cpu_ops_oss__srcs
+      _extension_data_loader__srcs
+      _extension_module__srcs
+      _extension_runner_util__srcs
+      _extension_llm_runner__srcs
+      _extension_tensor__srcs
+      _extension_threadpool__srcs
+      _extension_training__srcs
+      _train_xor__srcs
+      _executor_runner__srcs
+      _size_test__srcs
+      _mps_executor_runner__srcs
+      _mps_backend__srcs
+      _mps_schema__srcs
+      _xnn_executor_runner__srcs
+      _xnnpack_backend__srcs
+      _xnnpack_schema__srcs
+      _vulkan_schema__srcs
+      _custom_ops__srcs
+      _llama_runner__srcs
+  )
+  foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS
+                                  BUILD_VARIABLES_VARNAMES
+  )
+    if("${filelist_and_varname_1}" STREQUAL "_custom_ops__srcs")
+      continue()
+    endif()
+    append_filelist(
+      ${filelist_and_varname_0}
+      "${filelist_and_varname_1}_from_build_variables"
+    )
+    if(NOT ${filelist_and_varname_1} STREQUAL
+       ${filelist_and_varname_1}_from_build_variables
+    )
+      message(
+        FATAL_ERROR
+          "Buck-generated ${filelist_and_varname_1} does not match hardcoded "
+          "${filelist_and_varname_0} in build_variables.bzl. Left: "
+          "${${filelist_and_varname_1}}\n "
+          "Right: ${${filelist_and_varname_1}_from_build_variables}"
+      )
+    endif()
+  endforeach()
+endfunction()

build/Test.cmake

@@ -36,6 +36,7 @@ function(et_cxx_test target_name)
   cmake_parse_arguments(ET_CXX_TEST "" "" "${multi_arg_names}" ${ARGN})
 
   add_executable(${target_name} ${ET_CXX_TEST_SOURCES} ${EXECUTORCH_ROOT}/runtime/core/exec_aten/testing_util/tensor_util.cpp)
+  find_package(GTest)
   # Includes gtest, gmock, executorch by default
   target_link_libraries(
     ${target_name} GTest::gtest GTest::gtest_main GTest::gmock executorch

build/Utils.cmake

@@ -329,29 +329,25 @@ function(resolve_python_executable)
 endfunction()
 
 # find_package(Torch CONFIG REQUIRED) replacement for targets that have a
-# header-only Torch dependency. Because find_package sets variables in the
-# parent scope, we use a macro to preserve this rather than maintaining our own
-# list of those variables.
-macro(find_package_torch_headers)
-  # We cannot simply use CMAKE_FIND_ROOT_PATH_BOTH, because that does not
-  # propagate into TorchConfig.cmake.
-  foreach(mode_kind IN ITEMS PACKAGE LIBRARY INCLUDE)
-    set(OLD_CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}
-        ${CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}}
-    )
-    set(CMAKE_FIND_ROOT_PATH_MODE_${mode_kind} BOTH)
-  endforeach()
-  find_package_torch()
-  foreach(mode_kind IN ITEMS PACKAGE LIBRARY INCLUDE)
-    set(CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}
-        ${OLD_CMAKE_FIND_ROOT_PATH_MODE_${mode_kind}}
-    )
-  endforeach()
-endmacro()
+# header-only Torch dependency.
+#
+# Unlike find_package(Torch ...), this will only set
+# TORCH_INCLUDE_DIRS in the parent scope. In particular, it will NOT
+# set any of the following:
+# - TORCH_FOUND
+# - TORCH_LIBRARY
+# - TORCH_CXX_FLAGS
+function(find_package_torch_headers)
+  # We implement this way rather than using find_package so that
+  # cross-compilation can still use the host's installed copy of
+  # torch, since the headers should be fine.
+  get_torch_base_path(TORCH_BASE_PATH)
+  set(TORCH_INCLUDE_DIRS "${TORCH_BASE_PATH}/include;${TORCH_BASE_PATH}/include/torch/csrc/api/include" PARENT_SCOPE)
+endfunction()
 
-# Add the Torch CMake configuration to CMAKE_PREFIX_PATH so that find_package
-# can find Torch.
-function(add_torch_to_cmake_prefix_path)
+# Return the base path to the installed Torch Python library in
+# outVar.
+function(get_torch_base_path outVar)
   if(NOT PYTHON_EXECUTABLE)
     resolve_python_executable()
   endif()
@@ -370,6 +366,13 @@ function(add_torch_to_cmake_prefix_path)
     message("Output:\n${_tmp_torch_path}")
     message(FATAL_ERROR "Error:\n${_tmp_torch_path_error}")
   endif()
+  set(${outVar} ${_tmp_torch_path} PARENT_SCOPE)
+endfunction()
+
+# Add the Torch CMake configuration to CMAKE_PREFIX_PATH so that find_package
+# can find Torch.
+function(add_torch_to_cmake_prefix_path)
+  get_torch_base_path(_tmp_torch_path)
   list(APPEND CMAKE_PREFIX_PATH "${_tmp_torch_path}")
   set(CMAKE_PREFIX_PATH
       "${CMAKE_PREFIX_PATH}"

build/build_android_llm_demo.sh

@@ -44,6 +44,7 @@ build_android_native_library() {
     -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
     -DANDROID_ABI="${ANDROID_ABI}" \
     -DANDROID_PLATFORM=android-26 \
+    -DBUILD_TESTING=OFF \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \
     -DEXECUTORCH_BUILD_XNNPACK=ON \
@@ -73,6 +74,7 @@ build_android_native_library() {
     -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
     -DANDROID_ABI="${ANDROID_ABI}" \
     -DANDROID_PLATFORM=android-26 \
+    -DBUILD_TESTING=OFF \
     -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \