Remove comments

Author: pcuenca

Sources/Tokenizers/BPETokenizer.swift

@@ -163,7 +163,6 @@ class BPETokenizer: PreTrainedTokenizerModel {
         let bpeTokens = self.bpe(token: text).split(separator: " ").map { String($0) }
         for token in bpeTokens {
             if convertTokenToId(token) != unknownTokenId {
-//            if let _ = tokensToIds[token as NSString] {
                 tokens.append(token)
             } else {
                 // TODO: if config.byte_fallback is False, append the unknown token instead

Sources/Tokenizers/Tokenizer.swift

@@ -183,15 +183,6 @@ public class PreTrainedTokenizer: Tokenizer {
         }.joined(separator: "|")
         addedTokensRegex = try? NSRegularExpression(pattern: addedTokensRegexString, options: [])
 
-//        let addedTokensRegexString = (tokenizerData.addedTokens?.arrayValue ?? []).compactMap { addedToken in
-//            guard let content = addedToken.content?.stringValue else { return nil }
-//            let prefix = (addedToken.lstrip?.boolValue ?? false ? #"\s*"# : "")
-//            let suffix = (addedToken.rstrip?.boolValue ?? false ? #"\s*"# : "")
-//            let token = NSRegularExpression.escapedPattern(for: content)
-//            return "\(prefix)(\(token))\(suffix)"
-//        }.joined(separator: "|")
-//        addedTokensRegex = try? NSRegularExpression(pattern: addedTokensRegexString, options: [])
-
         // TODO: specialTokens are stored but never used
         self.specialTokens = specialTokens
         self.addedTokens = Set(addedTokens.keys)

Tests/TokenizersTests/TokenizerTests.swift

@@ -70,10 +70,6 @@ class GemmaTokenizerTests: TokenizerTests {
         let cases = ["à" /* 0x61 0x300 */, "à" /* 0xe0 */]
         let expected = [217138, 1305]
 
-//        for x in cases.map { $0.unicodeScalars.map { String(format:"0x%lX", $0.value) } } {
-//            print(x)
-//        }
-
         // These are different characters
         for (s, expected) in zip(cases, expected) {
             let encoded = await tester.tokenizer?.encode(text: " " + s)