Fix pip install

Author: larryliu0820

.github/workflows/trunk.yml

@@ -39,6 +39,5 @@ jobs:
         ${CONDA_RUN} pip install . -v
         ${CONDA_RUN} pip install pytest
 
-
         # Run tests
         ${CONDA_RUN} pytest

CMakeLists.txt

@@ -131,6 +131,7 @@ if(TOKENIZERS_BUILD_PYTHON)
     GIT_TAG v2.13.6
   )
   FetchContent_MakeAvailable(pybind11)
+
   # Create the Python extension module
   pybind11_add_module(pytorch_tokenizers_cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/python_bindings.cpp
@@ -142,26 +143,28 @@ if(TOKENIZERS_BUILD_PYTHON)
   # Set properties for the Python extension
   target_compile_definitions(pytorch_tokenizers_cpp PRIVATE VERSION_INFO=${PROJECT_VERSION})
 
-  # Set the output name - let setuptools handle the output directory
+  # Set the output name and let setuptools control the output directory
   set_target_properties(pytorch_tokenizers_cpp PROPERTIES
     OUTPUT_NAME "pytorch_tokenizers_cpp"
   )
 
   # Don't install the Python extension here - let setuptools handle it
-  # The install() command is not needed for Python extensions built via setuptools
+  # The setup.py will copy the built extension to the appropriate location
 endif()
 
 # Installation rules
 include(GNUInstallDirs)
 
-# Install the library and its dependencies
-install(
-  TARGETS tokenizers re2 sentencepiece-static
-  EXPORT tokenizers-targets
-  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-)
+if(NOT TOKENIZERS_BUILD_PYTHON)
+  # Install the library and its dependencies
+  install(
+    TARGETS tokenizers re2 sentencepiece-static
+    EXPORT tokenizers-targets
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+  )
+endif()
 
 # Install header files
 install(

pytest.ini

@@ -1,21 +1,24 @@
 [pytest]
 # Pytest configuration for PyTorch Tokenizers
 
-# Test discovery - be very explicit about what to test
+# Test discovery
 testpaths = test
 python_files = test_*.py *_test.py
 python_classes = Test*
 python_functions = test_*
 
-# Add the project root to Python path
-pythonpath = .
-
 # Output options with explicit ignores
 addopts =
-    -v
-    --tb=short
-    --strict-markers
-    --disable-warnings
+    # show summary of all tests that did not pass
+    -rEfX
+    # Make tracebacks shorter
+    --tb=native
+    # capture only Python print and C++ py::print, but not C output (low-level Python errors)
+    --capture=sys
+    # don't suppress warnings, but don't shove them all to the end either
+    -p no:warnings
+    # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
+    # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
     --ignore=third-party
     --ignore=build
     --ignore=cmake

setup.py

@@ -52,7 +52,7 @@ def build_extension(self, ext):
             "-DTOKENIZERS_BUILD_PYTHON=ON",
             "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
         ]
-        build_args = []
+        build_args = ["--target", "pytorch_tokenizers_cpp"]
 
         # Adding CMake arguments set as environment variable
         # (needed e.g. to build for ARM OSX on conda-forge)
@@ -133,7 +133,7 @@ def build_extension(self, ext):
     ext_modules=[CMakeExtension("pytorch_tokenizers_cpp")],
     cmdclass={"build_ext": CMakeBuild},
     zip_safe=False,
-    python_requires=">=3.8",
+    python_requires=">=3.10",
     install_requires=[
         "pybind11>=2.6.0",
     ],