We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 4f91561 commit 508c540Copy full SHA for 508c540
Sources/Tokenizers/Tokenizer.swift
@@ -68,7 +68,7 @@ struct TokenizerModel {
68
"LlamaTokenizer" : LlamaTokenizer.self,
69
"T5Tokenizer" : T5Tokenizer.self,
70
"WhisperTokenizer" : WhisperTokenizer.self,
71
-
+ "CohereTokenizer" : CohereTokenizer.self,
72
"PreTrainedTokenizer": BPETokenizer.self
73
]
74
@@ -267,5 +267,6 @@ class CodeGenTokenizer : BPETokenizer {}
267
class WhisperTokenizer : BPETokenizer {}
268
class GemmaTokenizer : BPETokenizer {}
269
class CodeLlamaTokenizer: BPETokenizer {}
270
+class CohereTokenizer : BPETokenizer {}
271
272
class T5Tokenizer : UnigramTokenizer {}
0 commit comments