Skip to content

Commit 290cc0a

Browse files
committed
Remove duplicate function
1 parent 767e8e6 commit 290cc0a

File tree

1 file changed

+0
-9
lines changed

1 file changed

+0
-9
lines changed

Tests/TokenizersTests/AddedTokensTests.swift

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,6 @@ class AddedTokensTests: XCTestCase {
2828
XCTAssertEqual(decoded, "<bos>This\n\nis\na\ntest.")
2929
}
3030

31-
func testGemmaAddedTokens() async throws {
32-
let tokenizer = try await AutoTokenizer.from(pretrained: "pcuenq/gemma-tokenizer")
33-
let inputIds = tokenizer("This\n\nis\na\ntest.")
34-
XCTAssertEqual(inputIds, [2, 1596, 109, 502, 108, 235250, 108, 2195, 235265])
35-
36-
let decoded = tokenizer.decode(tokens: inputIds)
37-
XCTAssertEqual(decoded, "<bos>This\n\nis\na\ntest.")
38-
}
39-
4031
func testSplitWithCaptureGroups() {
4132
let addedTokensRegexp = #"(<\|end\|>)\s*|(<\|raw\|>)\s*"#
4233
let captureRegex = try! NSRegularExpression(pattern: addedTokensRegexp, options: [])

0 commit comments

Comments
 (0)