Add the canonical cached token location (#170)

* Add the canonical cached token location.

I think ~/.huggingface is deprecated.

* Local fromPretrained test

Author: pcuenca

Sources/Hub/HubApi.swift

@@ -52,6 +52,7 @@ private extension HubApi {
                     )
                 }
             },
+            { try? String(contentsOf: .homeDirectory.appendingPathComponent(".cache/huggingface/token"), encoding: .utf8) },
             { try? String(contentsOf: .homeDirectory.appendingPathComponent(".huggingface/token"), encoding: .utf8) }
         ]
         return possibleTokens

Tests/TokenizersTests/TokenizerTests.swift

@@ -140,6 +140,23 @@ class LlamaPostProcessorOverrideTests: XCTestCase {
     }
 }
 
+class LocalFromPretrainedTests: XCTestCase {
+    func testLocalTokenizerFromPretrained() async throws {
+        let downloadDestination: URL = {
+            let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first!
+            return base.appending(component: "hf-local-pretrained-tests-downloads")
+        }()
+
+        let hubApi = HubApi(downloadBase: downloadDestination)
+        let downloadedTo = try await hubApi.snapshot(from: "pcuenq/gemma-tokenizer")
+
+        let tokenizer = try await AutoTokenizer.from(modelFolder: downloadedTo) as? PreTrainedTokenizer
+        XCTAssertNotNil(tokenizer)
+
+        try FileManager.default.removeItem(at: downloadDestination)
+    }
+}
+
 class BertDiacriticsTests: XCTestCase {
     func testBertCased() async throws {
         guard let tokenizer = try await AutoTokenizer.from(pretrained: "distilbert/distilbert-base-multilingual-cased") as? PreTrainedTokenizer else {