Skip to content

Commit cf385ca

Browse files
authored
Fix docstring, type annotation for private method (#19)
1 parent 40d9b1f commit cf385ca

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

tiktoken/core.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> lis
104104
This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster).
105105
106106
```
107-
>>> enc.encode_batch(["hello world", "goodbye world"])
107+
>>> enc.encode_ordinary_batch(["hello world", "goodbye world"])
108108
[[31373, 995], [11274, 16390, 995]]
109109
```
110110
"""
@@ -285,7 +285,7 @@ def _encode_single_piece(self, text_or_bytes: Union[str, bytes]) -> list[int]:
285285
text_or_bytes = text_or_bytes.encode("utf-8")
286286
return self._core_bpe.encode_single_piece(text_or_bytes)
287287

288-
def _encode_only_native_bpe(self, text: str) -> list[str]:
288+
def _encode_only_native_bpe(self, text: str) -> list[int]:
289289
"""Encodes a string into tokens, but do regex splitting in Python."""
290290
_unused_pat = regex.compile(self._pat_str)
291291
ret = []

0 commit comments

Comments
 (0)