Try again

Author: larryliu0820

.github/workflows/pull.yml

@@ -30,6 +30,7 @@ jobs:
 
         # Install tokenizers
         pip install . -v
+        pip install pytest
 
         # Run tests
         pytest

.github/workflows/trunk.yml

@@ -37,6 +37,8 @@ jobs:
 
         # Install tokenizers
         ${CONDA_RUN} pip install . -v
+        ${CONDA_RUN} pip install pytest
+
 
         # Run tests
         ${CONDA_RUN} pytest

CMakeLists.txt

@@ -142,16 +142,13 @@ if(TOKENIZERS_BUILD_PYTHON)
   # Set properties for the Python extension
   target_compile_definitions(pytorch_tokenizers_cpp PRIVATE VERSION_INFO=${PROJECT_VERSION})
 
-  # Set the output name and location
+  # Set the output name - let setuptools handle the output directory
   set_target_properties(pytorch_tokenizers_cpp PROPERTIES
     OUTPUT_NAME "pytorch_tokenizers_cpp"
   )
 
-  # Install the Python extension
-  install(TARGETS pytorch_tokenizers_cpp
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-  )
+  # Don't install the Python extension here - let setuptools handle it
+  # The install() command is not needed for Python extensions built via setuptools
 endif()
 
 # Installation rules

setup.py

@@ -29,6 +29,9 @@ class CMakeBuild(build_ext):
     def build_extension(self, ext):
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
 
+        # Ensure the extension goes into the pytorch_tokenizers package directory
+        extdir = os.path.join(extdir, "pytorch_tokenizers")
+
         # Required for auto-detection & inclusion of auxiliary "native" libs
         if not extdir.endswith(os.path.sep):
             extdir += os.path.sep