Export files required by OPT to load the tokenizer (#1571)

* initial commit

* include Tuans proposal

* Update src/sparseml/transformers/export.py

Author: dbogunowicz

src/sparseml/transformers/export.py

@@ -91,13 +91,18 @@
 __all__ = ["export_transformer_to_onnx", "load_task_model"]
 
 MODEL_ONNX_NAME = "model.onnx"
-EXTERNAL_ONNX_DATA_NAME = "model.data"
-MANDATORY_DEPLOYMENT_FILES: List[str] = [
+MANDATORY_DEPLOYMENT_FILES = [
     MODEL_ONNX_NAME,
     "tokenizer_config.json",
     "config.json",
 ]
-OPTIONAL_DEPLOYMENT_FILES: List[str] = [EXTERNAL_ONNX_DATA_NAME, "tokenizer.json"]
+EXTERNAL_ONNX_DATA_NAME = ["model.data"]
+OPT_TOKENIZER_FILES = ["special_tokens_map.json", "vocab.json", "merges.txt"]
+
+OPTIONAL_DEPLOYMENT_FILES: List[str] = ["tokenizer.json"]
+OPTIONAL_DEPLOYMENT_FILES.extend(EXTERNAL_ONNX_DATA_NAME)
+OPTIONAL_DEPLOYMENT_FILES.extend(OPT_TOKENIZER_FILES)
+
 
 _LOGGER = logging.getLogger(__name__)