Skip to content

Commit e5b6ce2

Browse files
committed
chore: format
1 parent 0bb1a9d commit e5b6ce2

File tree

2 files changed

+6
-6
lines changed

2 files changed

+6
-6
lines changed

Sources/Tokenizers/BertTokenizer.swift

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,14 @@ public class BertTokenizer {
4545
self.fuseUnknownTokens = fuseUnknownTokens
4646
}
4747

48-
public required convenience init(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String : Int]) throws {
48+
public required convenience init(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String: Int]) throws {
4949
guard var vocab = tokenizerData.model?.vocab?.dictionary as? [String: Int] else { throw TokenizerError.missingVocab }
5050
if let addedTokens = tokenizerData.added_tokens?.dictionary["value"] as? [[String: Any]],
51-
let pairs = addedTokens.compactMap({ ($0["content"] as? String, $0["id"] as? Int) }) as? [(String, Int)] {
52-
vocab.merge(pairs, uniquingKeysWith: {$1})
51+
let pairs = addedTokens.compactMap({ ($0["content"] as? String, $0["id"] as? Int) }) as? [(String, Int)]
52+
{
53+
vocab.merge(pairs, uniquingKeysWith: { $1 })
5354
}
54-
vocab.merge(addedTokens, uniquingKeysWith: {$1})
55+
vocab.merge(addedTokens, uniquingKeysWith: { $1 })
5556
let merges = tokenizerData.model?.merges?.value as? [String]
5657
let tokenizeChineseChars = tokenizerConfig.handleChineseChars?.boolValue ?? true
5758
let eosToken = tokenizerConfig.eosToken?.stringValue

Tests/TokenizersTests/BertTokenizerTests.swift

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,8 @@
66
// Copyright © 2019 Hugging Face. All rights reserved.
77
//
88

9-
@testable import Tokenizers
109
@testable import Hub
11-
10+
@testable import Tokenizers
1211

1312
class BertTokenizerTests: XCTestCase {
1413
override func setUp() {

0 commit comments

Comments
 (0)