Skip to content

Commit 508c540

Browse files
authored
add CohereTokenizer (#80)
1 parent 4f91561 commit 508c540

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

Sources/Tokenizers/Tokenizer.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ struct TokenizerModel {
6868
"LlamaTokenizer" : LlamaTokenizer.self,
6969
"T5Tokenizer" : T5Tokenizer.self,
7070
"WhisperTokenizer" : WhisperTokenizer.self,
71-
71+
"CohereTokenizer" : CohereTokenizer.self,
7272
"PreTrainedTokenizer": BPETokenizer.self
7373
]
7474

@@ -267,5 +267,6 @@ class CodeGenTokenizer : BPETokenizer {}
267267
class WhisperTokenizer : BPETokenizer {}
268268
class GemmaTokenizer : BPETokenizer {}
269269
class CodeLlamaTokenizer: BPETokenizer {}
270+
class CohereTokenizer : BPETokenizer {}
270271

271272
class T5Tokenizer : UnigramTokenizer {}

0 commit comments

Comments
 (0)