Deepseek R1 tokenization support and fixes (#159)

* Update Jinja, add Qwen R1 test

* More robust added token support

Serialized AddedToken class partially supported (in addition to String
values)

* Actually pass added tokens to decoder

* Temporarily revert jinja upgrade

Author: pcuenca

Sources/Tokenizers/BPETokenizer.swift

@@ -83,11 +83,11 @@ class BPETokenizer: PreTrainedTokenizerModel {
             self.unknownToken = nil
             self.unknownTokenId = nil
         }
-        
-        eosToken = tokenizerConfig.eosToken?.stringValue
+
+        eosToken = addedTokenAsString(tokenizerConfig.eosToken)
         eosTokenId = eosToken == nil ? nil : tokensToIds[eosToken! as NSString]
 
-        bosToken = tokenizerConfig.bosToken?.stringValue
+        bosToken = addedTokenAsString(tokenizerConfig.bosToken)
         bosTokenId = bosToken == nil ? nil : tokensToIds[bosToken! as NSString]
 
         fuseUnknownTokens = tokenizerConfig.fuseUnk?.boolValue ?? false

Sources/Tokenizers/Tokenizer.swift

@@ -41,6 +41,17 @@ public protocol TokenizingModel {
     var fuseUnknownTokens: Bool { get }
 }
 
+// Helper - possibly to be moved somewhere else
+func addedTokenAsString(_ addedToken: Config?) -> String? {
+    guard let addedToken = addedToken else { return nil }
+    if let stringValue = addedToken.stringValue {
+        return stringValue
+    }
+    // This is possibly a serialization of the AddedToken class
+    // TODO: support lstrip, rstrip, normalized, etc.
+    return addedToken.content?.stringValue
+}
+
 public extension TokenizingModel {
     func callAsFunction(_ text: String) -> [String] {
         tokenize(text: text)
@@ -241,7 +252,7 @@ public class PreTrainedTokenizer: Tokenizer {
         self.preTokenizer = PreTokenizerFactory.fromConfig(config: tokenizerData.preTokenizer)
         self.normalizer = NormalizerFactory.fromConfig(config: tokenizerData.normalizer)
         self.postProcessor = PostProcessorFactory.fromConfig(config: tokenizerData.postProcessor)
-        self.decoder = DecoderFactory.fromConfig(config: tokenizerData.decoder)
+        self.decoder = DecoderFactory.fromConfig(config: tokenizerData.decoder, addedTokens: self.addedTokens)
         self.cleanUpTokenizationSpaces = tokenizerConfig.cleanUpTokenizationSpaces?.boolValue ?? true
         self.tokenizerConfig = tokenizerConfig
 
@@ -421,7 +432,13 @@ public class PreTrainedTokenizer: Tokenizer {
         // TODO: maybe keep NSString here
         for (key, value) in tokenizerConfig.dictionary as [String : Any] {
             if specialTokenAttributes.contains(key), !(value is NSNull) {
-                context[key] = value
+                if let stringValue = value as? String {
+                    context[key] = stringValue
+                } else if let dictionary = value as? [NSString:Any] {
+                    context[key] = addedTokenAsString(Config(dictionary))
+                } else {
+                    context[key] = value
+                }
             }
         }