Enabling static interpreter embedding for manylinux. (#1064)

* Removing dead file.

* Checking that we can distribute with static python embedding for

manylinux

* Many linux embed interpreter.

* Building wheels manylinux with static embedding

* Better script.

* typo.

* Using a dummy feature?

* default features ?

* Back into order.

* Fixing manylinux ??.

* Local dir.

* Missing star.

* Makedir ?

* Monkey coding this.

* extension module ?

* Building with default features `RustExtension`.

* bdist_wheel + rustextension any better ?

* update rust-py version.

* Forcing extension module.

* No default features.

* Remove py37 out of spite

* Revert "Remove py37 out of spite"

This reverts commit 6ab7fac.

* Really extraneous feature.

* Fix build wheels.

* Putting things back in place.

Author: Narsil

bindings/python/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-python"
-version = "0.11.0"
+version = "0.13.0"
 authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
 edition = "2018"
 
@@ -14,7 +14,7 @@ serde = { version = "1.0", features = [ "rc", "derive" ]}
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.7.1"
-pyo3 = "0.16.2"
+pyo3 = { version = "0.16.2", features = ["extension-module"] }
 numpy = "0.16.2"
 ndarray = "0.13"
 onig = { version = "6.0", default-features = false }
@@ -26,7 +26,7 @@ path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.1"
-pyo3 = { version = "0.16.2", features = ["auto-initialize"] }
 
 [features]
-default = ["pyo3/extension-module"]
+test = ["pyo3/auto-initialize"]
+

bindings/python/Cargo.toml

@@ -20,7 +20,7 @@ TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json
 test: $(TESTS_RESOURCES)
 	pip install pytest requests setuptools_rust numpy pyarrow datasets
 	python -m pytest -s -v tests
-	cargo test --no-default-features
+	cargo test --features test
 
 $(DATA_DIR)/big.txt :
 	$(dir_guard)

bindings/python/Makefile

@@ -1,23 +1,28 @@
 #!/bin/bash
 set -ex
 
-curl https://sh.rustup.rs -sSf | sh -s -- -y
+if ! command -v cargo &> /dev/null
+then
+    curl https://sh.rustup.rs -sSf | sh -s -- -y
+fi
+
 export PATH="$HOME/.cargo/bin:$PATH"
 
-for PYBIN in /opt/python/{cp37-cp37m,cp38-cp38,cp39-cp39,cp310-cp310}/bin; do
+for PYBIN in /opt/python/cp{37,38,39,310}*/bin; do
     export PYTHON_SYS_EXECUTABLE="$PYBIN/python"
 
-    "${PYBIN}/pip" install -U setuptools-rust==0.11.3
+    "${PYBIN}/pip" install -U setuptools-rust setuptools wheel
     "${PYBIN}/python" setup.py bdist_wheel
     rm -rf build/*
 done
 
-for whl in dist/*.whl; do
+for whl in ./dist/*.whl; do
     auditwheel repair "$whl" -w dist/
 done
 
 # Keep only manylinux wheels
-rm dist/*-linux_*
+rm ./dist/*-linux_*
+
 
 # Upload wheels
 /opt/python/cp37-cp37m/bin/pip install -U awscli

bindings/python/build-wheels.sh

@@ -1,4 +1,4 @@
-__version__ = "0.13.0.dev0"
+__version__ = "0.13.0"
 
 from typing import Tuple, Union, Tuple, List
 from enum import Enum

bindings/python/py_src/tokenizers/__init__.py

@@ -8,7 +8,7 @@
 
 setup(
     name="tokenizers",
-    version="0.13.0.dev0",
+    version="0.13.0",
     description="Fast and Customizable Tokenizers",
     long_description=open("README.md", "r", encoding="utf-8").read(),
     long_description_content_type="text/markdown",