Add CodeLlamaTokenizer to known tokenizers (#50)

krzyzanowskim · web-flow · commit 15bc01ea0fc7 · 2024-02-28T18:53:09.000+01:00
diff --git a/Sources/Tokenizers/Tokenizer.swift b/Sources/Tokenizers/Tokenizer.swift
@@ -59,6 +59,7 @@ struct TokenizerModel {
         "GPT2Tokenizer"   : GPT2Tokenizer.self,
         "FalconTokenizer" : FalconTokenizer.self,
         "LlamaTokenizer"  : LlamaTokenizer.self,
+        "CodeLlamaTokenizer": LlamaTokenizer.self,
         "CodeGenTokenizer": CodeGenTokenizer.self,
         "WhisperTokenizer": WhisperTokenizer.self,
         "T5Tokenizer"     : T5Tokenizer.self,
