[Python] Add sentencepiece as installation requirement (#3030)

MasterJH5574 · web-flow · commit e462885c778c · 2024-11-15T11:15:03.000-05:00
This PR adds the sentencepiece package as a Python installation
requirement for tokenizer coverage when running `gen_config`.
diff --git a/python/setup.py b/python/setup.py
@@ -104,6 +104,7 @@ def main():
             "safetensors",
             "requests",
             "tqdm",
+            "sentencepiece",
             "tiktoken",
             "prompt_toolkit",
             "openai",
