Added new methods, fixed visionos build

Author: jkrukowski

Package.swift

@@ -10,6 +10,7 @@ let package = Package(
         .iOS(.v16),
         .tvOS(.v16),
         .watchOS(.v9),
+        .visionOS(.v1),
     ],
     products: [
         .executable(
@@ -34,13 +35,17 @@ let package = Package(
         .binaryTarget(
             name: "Sentencepiece",
             url:
-                "https://github.com/jkrukowski/swift-sentencepiece/releases/download/0.0.4/sentencepiece.xcframework.zip",
-            checksum: "9168a242ffc75cdecb3bd21f4671842bb48d583d1cbc83e5a255d3e348622680"
+                "https://github.com/jkrukowski/swift-sentencepiece/releases/download/0.0.5/sentencepiece.xcframework.zip",
+            checksum: "4b3b3fefc5ce55edd9fa8b0133b1027db19d3c2b27c63e993727aebe2b3545a8"
         ),
         .target(
             name: "SentencepieceTokenizer",
             dependencies: [
                 "Sentencepiece"
+            ],
+            linkerSettings: [
+                .linkedLibrary("c++"),
+                .linkedLibrary("stdc++")
             ]
         ),
         .testTarget(
@@ -50,11 +55,6 @@ let package = Package(
             ],
             resources: [
                 .copy("Model")
-            ],
-            linkerSettings: [
-                .unsafeFlags([
-                    "-lc++"
-                ])
             ]
         ),
     ]

Package@swift-5.9.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 6.0
+// swift-tools-version: 5.9
 // The swift-tools-version declares the minimum version of Swift required to build this package.
 
 import PackageDescription
@@ -10,6 +10,7 @@ let package = Package(
         .iOS(.v16),
         .tvOS(.v16),
         .watchOS(.v9),
+        .visionOS(.v1),
     ],
     products: [
         .executable(
@@ -34,13 +35,17 @@ let package = Package(
         .binaryTarget(
             name: "Sentencepiece",
             url:
-                "https://github.com/jkrukowski/swift-sentencepiece/releases/download/0.0.4/sentencepiece.xcframework.zip",
-            checksum: "9168a242ffc75cdecb3bd21f4671842bb48d583d1cbc83e5a255d3e348622680"
+                "https://github.com/jkrukowski/swift-sentencepiece/releases/download/0.0.5/sentencepiece.xcframework.zip",
+            checksum: "4b3b3fefc5ce55edd9fa8b0133b1027db19d3c2b27c63e993727aebe2b3545a8"
         ),
         .target(
             name: "SentencepieceTokenizer",
             dependencies: [
                 "Sentencepiece"
+            ],
+            linkerSettings: [
+                .linkedLibrary("c++"),
+                .linkedLibrary("stdc++")
             ]
         ),
         .testTarget(
@@ -50,11 +55,6 @@ let package = Package(
             ],
             resources: [
                 .copy("Model")
-            ],
-            linkerSettings: [
-                .unsafeFlags([
-                    "-lc++"
-                ])
             ]
         ),
     ]

Sources/SentencepieceTokenizer/SentencepieceTokenizer.swift

@@ -56,13 +56,18 @@ public final class SentencepieceTokenizer {
     }
 
     public func idToToken(_ id: Int) throws -> String {
+        precondition(id - tokenOffset >= 0)
         guard let tokenPtr = spm_id_to_piece(processor, Int32(id - tokenOffset)) else {
             throw Error.failedToProcess
         }
         defer { tokenPtr.deallocate() }
         return String(cString: tokenPtr)
     }
 
+    public func tokenToId(_ token: String) -> Int {
+        Int(spm_piece_to_id(processor, token)) + tokenOffset
+    }
+
     public func encode(_ text: String) throws -> [Int] {
         var size: Int32 = 0
         guard let encodedPtr = spm_encode(processor, text, &size) else {
@@ -73,6 +78,10 @@ public final class SentencepieceTokenizer {
         return result.map { Int($0) + tokenOffset }
     }
 
+    public func setEncodeExtraOptions(_ options: String) {
+        spm_set_encode_extra_options(processor, options)
+    }
+
     public func decode(_ ids: [Int]) throws -> String {
         let encoded = ids.map { Int32($0 - tokenOffset) }
         guard let decodedPtr = spm_decode(processor, encoded, Int32(encoded.count)) else {
@@ -81,4 +90,8 @@ public final class SentencepieceTokenizer {
         defer { decodedPtr.deallocate() }
         return String(cString: decodedPtr)
     }
+
+    public func setDecodeExtraOptions(_ options: String) {
+        spm_set_decode_extra_options(processor, options)
+    }
 }

Tests/SentencepieceTokenizerTests/SentencepieceTokenizerTests.swift

@@ -26,6 +26,11 @@ final class SentencepieceTokenizerTests: XCTestCase {
         XCTAssertEqual(try tokenizer.idToToken(8999), "▁world")
         XCTAssertEqual(try tokenizer.idToToken(38), "!")
 
+        XCTAssertEqual(tokenizer.tokenToId("▁Hello"), 35378)
+        XCTAssertEqual(tokenizer.tokenToId(","), 4)
+        XCTAssertEqual(tokenizer.tokenToId("▁world"), 8999)
+        XCTAssertEqual(tokenizer.tokenToId("!"), 38)
+
         XCTAssertEqual(try tokenizer.decode([]), "")
         XCTAssertEqual(try tokenizer.encode(""), [])
 
@@ -34,4 +39,24 @@ final class SentencepieceTokenizerTests: XCTestCase {
         XCTAssertEqual(tokenizer.bosTokenId, 2)
         XCTAssertEqual(tokenizer.eosTokenId, 3)
     }
+
+    func testSentencepieceEncodeExtraOptions() throws {
+        let modelPath = try XCTUnwrap(
+            Bundle.module.path(
+                forResource: "sentencepiece.bpe", ofType: "model", inDirectory: "Model"))
+        let tokenizer = try SentencepieceTokenizer(modelPath: modelPath)
+        tokenizer.setEncodeExtraOptions("reverse:bos:eos")
+        let output = try tokenizer.encode("Hello, world!")
+        XCTAssertEqual(output, [2, 38, 8999, 4, 35378, 3])
+    }
+
+    func testSentencepieceDecodeExtraOptions() throws {
+        let modelPath = try XCTUnwrap(
+            Bundle.module.path(
+                forResource: "sentencepiece.bpe", ofType: "model", inDirectory: "Model"))
+        let tokenizer = try SentencepieceTokenizer(modelPath: modelPath)
+        tokenizer.setDecodeExtraOptions("reverse:bos:eos")
+        let output = try tokenizer.decode([2, 35378, 4, 8999, 38, 3])
+        XCTAssertEqual(output, "! world, Hello")
+    }
 }