Skip to content

Commit 8a83416

Browse files
authored
Upgrade jinja (#160)
* Upgrade jinja * Deepseek chat template test
1 parent 1fab24c commit 8a83416

File tree

2 files changed

+12
-1
lines changed

2 files changed

+12
-1
lines changed

Package.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ let package = Package(
1313
],
1414
dependencies: [
1515
.package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.4.0"),
16-
.package(url: "https://github.com/maiqingqiang/Jinja", from: "1.0.6")
16+
.package(url: "https://github.com/johnmai-dev/Jinja", from: "1.1.0")
1717
],
1818
targets: [
1919
.executableTarget(

Tests/TokenizersTests/ChatTemplateTests.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,17 @@ class ChatTemplateTests: XCTestCase {
2424
XCTAssertEqual(decoded, decodedTarget)
2525
}
2626

27+
func testDeepSeekQwenChatTemplate() async throws {
28+
let tokenizer = try await AutoTokenizer.from(pretrained: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
29+
let encoded = try tokenizer.applyChatTemplate(messages: messages)
30+
let encodedTarget = [151646, 151644, 74785, 279, 23670, 15473, 4128, 13, 151645]
31+
XCTAssertEqual(encoded, encodedTarget)
32+
33+
let decoded = tokenizer.decode(tokens: encoded)
34+
let decodedTarget = "<|begin▁of▁sentence|><|User|>Describe the Swift programming language.<|Assistant|>"
35+
XCTAssertEqual(decoded, decodedTarget)
36+
}
37+
2738
func testDefaultTemplateFromArrayInConfig() async throws {
2839
let tokenizer = try await AutoTokenizer.from(pretrained: "mlx-community/Mistral-7B-Instruct-v0.3-4bit")
2940
let encoded = try tokenizer.applyChatTemplate(messages: messages)

0 commit comments

Comments
 (0)