Upgrade jinja (#160)

* Upgrade jinja * Deepseek chat template test
huggingface · Jan 24, 2025 · 8a83416 · 8a83416
1 parent 1fab24c
commit 8a83416
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/Package.swift b/Package.swift
@@ -13,7 +13,7 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.4.0"),
-        .package(url: "https://github.com/maiqingqiang/Jinja", from: "1.0.6")
+        .package(url: "https://github.com/johnmai-dev/Jinja", from: "1.1.0")
     ],
     targets: [
         .executableTarget(

diff --git a/Tests/TokenizersTests/ChatTemplateTests.swift b/Tests/TokenizersTests/ChatTemplateTests.swift
@@ -24,6 +24,17 @@ class ChatTemplateTests: XCTestCase {
         XCTAssertEqual(decoded, decodedTarget)
     }
 
+    func testDeepSeekQwenChatTemplate() async throws {
+        let tokenizer = try await AutoTokenizer.from(pretrained: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
+        let encoded = try tokenizer.applyChatTemplate(messages: messages)
+        let encodedTarget = [151646, 151644, 74785, 279, 23670, 15473, 4128, 13, 151645]
+        XCTAssertEqual(encoded, encodedTarget)
+
+        let decoded = tokenizer.decode(tokens: encoded)
+        let decodedTarget = "<｜begin▁of▁sentence｜><｜User｜>Describe the Swift programming language.<｜Assistant｜>"
+        XCTAssertEqual(decoded, decodedTarget)
+    }
+
     func testDefaultTemplateFromArrayInConfig() async throws {
         let tokenizer = try await AutoTokenizer.from(pretrained: "mlx-community/Mistral-7B-Instruct-v0.3-4bit")
         let encoded = try tokenizer.applyChatTemplate(messages: messages)