From 973057a8795e7d1b4bc9a343ade6693091caf790 Mon Sep 17 00:00:00 2001 From: Joseph Stahl <1269177+josephst@users.noreply.github.com> Date: Sun, 17 Mar 2024 15:19:10 -0400 Subject: [PATCH 1/3] Symlink to /usr/bin/xcrun so that `xcrun` binary is usable during build (used for compiling Metal shaders) Fixes https://github.com/ggerganov/llama.cpp/issues/6117 --- .devops/nix/package.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 76d96e63cd6eb..f93b25b240e3e 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -4,6 +4,7 @@ config, stdenv, mkShell, + runCommand, cmake, ninja, pkg-config, @@ -87,6 +88,11 @@ let ] ); + xcrunHost = runCommand "xcrunHost" {} '' + mkdir -p $out/bin + ln -s /usr/bin/xcrun $out/bin + ''; + # apple_sdk is supposed to choose sane defaults, no need to handle isAarch64 # separately darwinBuildInputs = @@ -157,6 +163,14 @@ effectiveStdenv.mkDerivation ( substituteInPlace ./*.py --replace "/usr/bin/env python" "${llama-python}/bin/python" ''; + # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015, + # `default.metallib` is compiled with Metal compiler from XCode + # and we need to escape sandbox on MacOS to access Metal compiler. + # `xcrun` is used find the path of the Metal compiler, which is varible + # and not on $PATH + # see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion + __noChroot = effectiveStdenv.isDarwin && useMetalKit; + nativeBuildInputs = [ cmake @@ -173,6 +187,8 @@ effectiveStdenv.mkDerivation ( ] ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static + ] ++ optionals (effectiveStdenv.isDarwin && useMetalKit) [ + xcrunHost ]; buildInputs = From 0af5c68a0ca2f63632ad61c6a388ddf6a7d563d4 Mon Sep 17 00:00:00 2001 From: Joseph Stahl <1269177+josephst@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:08:28 -0400 Subject: [PATCH 2/3] cmake - copy default.metallib to install directory When metal files are compiled to default.metallib, Cmake needs to add this to the install directory so that it's visible to llama-cpp Also, update package.nix to use absolute path for default.metallib (it's not finding the bundle) --- .devops/nix/package.nix | 2 ++ CMakeLists.txt | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index f93b25b240e3e..2d64a6808d66c 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -156,6 +156,8 @@ effectiveStdenv.mkDerivation ( postPatch = '' substituteInPlace ./ggml-metal.m \ --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";" + substituteInPlace ./ggml-metal.m \ + --replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";" # TODO: Package up each Python script or service appropriately. # If we were to migrate to buildPythonPackage and prepare the `pyproject.toml`, diff --git a/CMakeLists.txt b/CMakeLists.txt index b25cfd2fc0e17..d101d3a58e718 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1260,6 +1260,12 @@ if (LLAMA_METAL) GROUP_READ WORLD_READ DESTINATION ${CMAKE_INSTALL_BINDIR}) + if (NOT LLAMA_METAL_EMBED_LIBRARY) + install( + FILES ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + endif() endif() # From 4af8617e8125e52f362fa921cffc56e165423cef Mon Sep 17 00:00:00 2001 From: Joseph Stahl <1269177+josephst@users.noreply.github.com> Date: Sat, 23 Mar 2024 09:07:05 -0400 Subject: [PATCH 3/3] add `precompileMetalShaders` flag (defaults to false) to disable precompilation of metal shader Precompilation requires Xcode to be installed and requires disable sandbox on nix-darwin --- .devops/nix/package.nix | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 2d64a6808d66c..3fad5134ee4b2 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -36,7 +36,8 @@ # It's necessary to consistently use backendStdenv when building with CUDA support, # otherwise we get libstdc++ errors downstream. effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv, - enableStatic ? effectiveStdenv.hostPlatform.isStatic + enableStatic ? effectiveStdenv.hostPlatform.isStatic, + precompileMetalShaders ? false }@inputs: let @@ -166,12 +167,12 @@ effectiveStdenv.mkDerivation ( ''; # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015, - # `default.metallib` is compiled with Metal compiler from XCode + # `default.metallib` may be compiled with Metal compiler from XCode # and we need to escape sandbox on MacOS to access Metal compiler. # `xcrun` is used find the path of the Metal compiler, which is varible # and not on $PATH # see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion - __noChroot = effectiveStdenv.isDarwin && useMetalKit; + __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders; nativeBuildInputs = [ @@ -189,7 +190,7 @@ effectiveStdenv.mkDerivation ( ] ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static - ] ++ optionals (effectiveStdenv.isDarwin && useMetalKit) [ + ] ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]; @@ -235,7 +236,10 @@ effectiveStdenv.mkDerivation ( # Should likely use `rocmPackages.clr.gpuTargets`. "-DAMDGPU_TARGETS=gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102" ] - ++ optionals useMetalKit [ (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1") ]; + ++ optionals useMetalKit [ + (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1") + (cmakeBool "LLAMA_METAL_EMBED_LIBRARY" (!precompileMetalShaders)) + ]; # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level, # if they haven't been added yet.