Match token value with array and tuple

  * Add match cases for token value
  * Add BartTokenizer to known tokenizers

Author: shavit

Sources/Hub/Hub.swift

@@ -100,7 +100,14 @@ public struct Config {
     }
 
     /// Tuple of token identifier and string value
-    public var tokenValue: (UInt, String)? { value as? (UInt, String) }
+    public var tokenValue: (UInt, String)? { 
+        switch value {
+        case let (i, t) as (UInt, String): return (i, t)
+        case let (t, i) as (String, UInt): return (i, t)
+        case let a as [Any] where a.count == 2: return (a[1], a[0]) as? (UInt, String)
+        default: return nil
+        }
+    }
 }
 
 public class LanguageModelConfigurationFromHub {

Sources/Tokenizers/Tokenizer.swift

@@ -77,6 +77,7 @@ public protocol PreTrainedTokenizerModel: TokenizingModel {
 
 struct TokenizerModel {
     static let knownTokenizers: [String : PreTrainedTokenizerModel.Type] = [
+        "BartTokenizer"      : BertTokenizer.self,
         "BertTokenizer"      : BertTokenizer.self,
         "DistilbertTokenizer": BertTokenizer.self,
         "DistilBertTokenizer": BertTokenizer.self,