create and validate build_variables.bzl

First step of #8268 -- we are moving from buck-extracted filelist to using one shared filelist, and the first step is to create the shared filelist and validate it against the buck generation.

ghstack-source-id: 567f769
ghstack-comment-id: 2644414497
Pull Request resolved: #8326

Author: swolchok

CMakeLists.txt

@@ -44,6 +44,7 @@
 
 cmake_minimum_required(VERSION 3.19)
 project(executorch)
+include(build/Codegen.cmake)
 include(build/Utils.cmake)
 include(CMakeDependentOption)
 
@@ -396,6 +397,7 @@ if(NOT EXECUTORCH_SRCS_FILE)
   message(STATUS "executorch: Generating source lists")
   set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake")
   extract_sources(${EXECUTORCH_SRCS_FILE})
+  validate_build_variables()
 endif()
 
 # This file defines the `_<target>__srcs` variables used below.

build/Codegen.cmake

@@ -217,3 +217,127 @@ function(merge_yaml)
     WORKING_DIRECTORY ${EXECUTORCH_ROOT}
   )
 endfunction()
+
+# Append the file list in the variable named `name` in
+# build/build_variables.bzl to the variable named `outputvar` in the
+# caller's scope.
+function(append_filelist name outputvar)
+  # configure_file adds its input to the list of CMAKE_RERUN dependencies
+  configure_file(
+    ${PROJECT_SOURCE_DIR}/build/build_variables.bzl
+    ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY
+  )
+  execute_process(
+    COMMAND
+      "${PYTHON_EXECUTABLE}" -c
+      "exec(open('${PROJECT_SOURCE_DIR}/build/build_variables.bzl').read());print(';'.join(${name}))"
+    WORKING_DIRECTORY "${_rootdir}"
+    RESULT_VARIABLE _retval
+    OUTPUT_VARIABLE _tempvar
+    ERROR_VARIABLE _stderr
+  )
+  if(NOT _retval EQUAL 0)
+    message(
+      FATAL_ERROR
+        "Failed to fetch filelist ${name} from build_variables.bzl with output ${_tempvar} and stderr ${_stderr}"
+    )
+  endif()
+  string(REPLACE "\n" "" _tempvar "${_tempvar}")
+  list(APPEND ${outputvar} ${_tempvar})
+  set(${outputvar}
+      "${${outputvar}}"
+      PARENT_SCOPE
+  )
+endfunction()
+
+# Fail the build if the src lists in build_variables.bzl do not match
+# the src lists extracted from Buck and placed into
+# EXECUTORCH_SRCS_FILE. This is intended to be a safety mechanism
+# while we are in the process of removing Buck from the CMake build
+# and replacing it with build_variables.bzl; if you are seeing
+# failures after you have intentionally changed Buck srcs, then simply
+# update build_variables.bzl. If you are seeing failures after
+# changing something about the build system, make sure your changes
+# will work both before and after we finish replacing Buck with
+# build_variables.bzl, which should involve getting these lists to
+# match!
+function(validate_build_variables)
+  include(${EXECUTORCH_SRCS_FILE})
+  set(BUILD_VARIABLES_FILELISTS
+      EXECUTORCH_SRCS
+      EXECUTORCH_CORE_SRCS
+      PORTABLE_KERNELS_SRCS
+      OPTIMIZED_KERNELS_SRCS
+      QUANTIZED_KERNELS_SRCS
+      PROGRAM_SCHEMA_SRCS
+      OPTIMIZED_CPUBLAS_SRCS
+      OPTIMIZED_NATIVE_CPU_OPS_OSS_SRCS
+      EXTENSION_DATA_LOADER_SRCS
+      EXTENSION_MODULE_SRCS
+      EXTENSION_RUNNER_UTIL_SRCS
+      EXTENSION_LLM_RUNNER_SRCS
+      EXTENSION_TENSOR_SRCS
+      EXTENSION_THREADPOOL_SRCS
+      EXTENSION_TRAINING_SRCS
+      TRAIN_XOR_SRCS
+      EXECUTOR_RUNNER_SRCS
+      SIZE_TEST_SRCS
+      MPS_EXECUTOR_RUNNER_SRCS
+      MPS_BACKEND_SRCS
+      MPS_SCHEMA_SRCS
+      XNN_EXECUTOR_RUNNER_SRCS
+      XNNPACK_BACKEND_SRCS
+      XNNPACK_SCHEMA_SRCS
+      VULKAN_SCHEMA_SRCS
+      CUSTOM_OPS_SRCS
+      LLAMA_RUNNER_SRCS
+  )
+  set(BUILD_VARIABLES_VARNAMES
+      _executorch__srcs
+      _executorch_core__srcs
+      _portable_kernels__srcs
+      _optimized_kernels__srcs
+      _quantized_kernels__srcs
+      _program_schema__srcs
+      _optimized_cpublas__srcs
+      _optimized_native_cpu_ops_oss__srcs
+      _extension_data_loader__srcs
+      _extension_module__srcs
+      _extension_runner_util__srcs
+      _extension_llm_runner__srcs
+      _extension_tensor__srcs
+      _extension_threadpool__srcs
+      _extension_training__srcs
+      _train_xor__srcs
+      _executor_runner__srcs
+      _size_test__srcs
+      _mps_executor_runner__srcs
+      _mps_backend__srcs
+      _mps_schema__srcs
+      _xnn_executor_runner__srcs
+      _xnnpack_backend__srcs
+      _xnnpack_schema__srcs
+      _vulkan_schema__srcs
+      _custom_ops__srcs
+      _llama_runner__srcs
+  )
+  foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS
+                                  BUILD_VARIABLES_VARNAMES
+  )
+    append_filelist(
+      ${filelist_and_varname_0}
+      "${filelist_and_varname_1}_from_build_variables"
+    )
+    if(NOT ${filelist_and_varname_1} STREQUAL
+       ${filelist_and_varname_1}_from_build_variables
+    )
+      message(
+        FATAL_ERROR
+          "Buck-generated ${filelist_and_varname_1} does not match hardcoded "
+          "${filelist_and_varname_0} in build_variables.bzl. Left: "
+          "${${filelist_and_varname_1}}\n "
+          "Right: ${${filelist_and_varname_1}_from_build_variables}"
+      )
+    endif()
+  endforeach()
+endfunction()