Gemma tokenizer, fix Unicode split (#52)

* Add GemmaTokenizer class

* Fix unicode split, and do not omit empty sequences

This makes it equivalent to the Python and JavaScript versions as far as
I can tell.

Co-authored-by: davidkoski <davidkoski@users.noreply.github.com>

---------

Co-authored-by: davidkoski <davidkoski@users.noreply.github.com>

Author: pcuenca

Sources/Tokenizers/BPETokenizer.swift

@@ -46,7 +46,7 @@ class BPETokenizer: PreTrainedTokenizerModel {
         }
         var bpeRanks: Dictionary<BytePair, Int> = [:]
         for (i, item) in merges.enumerated() {
-            let tuple = item.split(separator: " ").map { String($0) }
+            let tuple = item.unicodeScalars.split(separator: " ", omittingEmptySubsequences: false).map { String($0) }
             let bp = BytePair(tuple: tuple)
             bpeRanks[bp] = i
         }

Sources/Tokenizers/Tokenizer.swift

@@ -55,16 +55,16 @@ public protocol PreTrainedTokenizerModel: TokenizingModel {
 
 struct TokenizerModel {
     static let knownTokenizers: [String : PreTrainedTokenizerModel.Type] = [
-        "BertTokenizer"   : BertTokenizer.self,
-        "GPT2Tokenizer"   : GPT2Tokenizer.self,
-        "FalconTokenizer" : FalconTokenizer.self,
-        "LlamaTokenizer"  : LlamaTokenizer.self,
-        "CodeLlamaTokenizer": LlamaTokenizer.self,
-        "CodeGenTokenizer": CodeGenTokenizer.self,
-        "WhisperTokenizer": WhisperTokenizer.self,
-        "T5Tokenizer"     : T5Tokenizer.self,
-
-        // Default
+        "BertTokenizer"      : BertTokenizer.self,
+        "CodeGenTokenizer"   : CodeGenTokenizer.self,
+        "CodeLlamaTokenizer" : CodeLlamaTokenizer.self,
+        "FalconTokenizer"    : FalconTokenizer.self,
+        "GemmaTokenizer"     : GemmaTokenizer.self,
+        "GPT2Tokenizer"      : GPT2Tokenizer.self,
+        "LlamaTokenizer"     : LlamaTokenizer.self,
+        "T5Tokenizer"        : T5Tokenizer.self,
+        "WhisperTokenizer"   : WhisperTokenizer.self,
+
         "PreTrainedTokenizer": BPETokenizer.self
     ]
 
@@ -245,10 +245,12 @@ extension AutoTokenizer {
 
 // MARK: - Tokenizer model classes
 
-class GPT2Tokenizer    : BPETokenizer {}
-class FalconTokenizer  : BPETokenizer {}
-class LlamaTokenizer   : BPETokenizer {}
-class CodeGenTokenizer : BPETokenizer {}
-class WhisperTokenizer : BPETokenizer {}
+class GPT2Tokenizer     : BPETokenizer {}
+class FalconTokenizer   : BPETokenizer {}
+class LlamaTokenizer    : BPETokenizer {}
+class CodeGenTokenizer  : BPETokenizer {}
+class WhisperTokenizer  : BPETokenizer {}
+class GemmaTokenizer    : BPETokenizer {}
+class CodeLlamaTokenizer: BPETokenizer {}
 
-class T5Tokenizer      : UnigramTokenizer {}
+class T5Tokenizer       : UnigramTokenizer {}