Skip to content

Commit 4a606f6

Browse files
authored
Merge pull request #1 from ensan-hcl/feat/minimum
minimum version of swift-transformers dedicated for tokenizer manipulation
2 parents abf5b16 + 407e518 commit 4a606f6

33 files changed

+5
-2651
lines changed

Package.swift

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,17 @@ let package = Package(
77
name: "swift-transformers",
88
platforms: [.iOS(.v16), .macOS(.v13)],
99
products: [
10-
.library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]),
11-
.executable(name: "transformers", targets: ["TransformersCLI"]),
12-
.executable(name: "hub-cli", targets: ["HubCLI"]),
10+
.library(name: "Transformers", targets: ["Tokenizers"]),
1311
],
1412
dependencies: [
1513
.package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.4.0")),
1614
.package(url: "https://github.com/johnmai-dev/Jinja", .upToNextMinor(from: "1.1.0"))
1715
],
1816
targets: [
19-
.executableTarget(
20-
name: "TransformersCLI",
21-
dependencies: [
22-
"Models", "Generation", "Tokenizers",
23-
.product(name: "ArgumentParser", package: "swift-argument-parser")]),
24-
.executableTarget(name: "HubCLI", dependencies: ["Hub", .product(name: "ArgumentParser", package: "swift-argument-parser")]),
2517
.target(name: "Hub", resources: [.process("FallbackConfigs")]),
2618
.target(name: "Tokenizers", dependencies: ["Hub", .product(name: "Jinja", package: "Jinja")]),
27-
.target(name: "TensorUtils"),
28-
.target(name: "Generation", dependencies: ["Tokenizers", "TensorUtils"]),
29-
.target(name: "Models", dependencies: ["Tokenizers", "Generation", "TensorUtils"]),
30-
.testTarget(name: "TokenizersTests", dependencies: ["Tokenizers", "Models", "Hub"], resources: [.process("Resources"), .process("Vocabs")]),
19+
.testTarget(name: "TokenizersTests", dependencies: ["Tokenizers", "Hub"], resources: [.process("Resources"), .process("Vocabs")]),
3120
.testTarget(name: "HubTests", dependencies: ["Hub"]),
3221
.testTarget(name: "PreTokenizerTests", dependencies: ["Tokenizers", "Hub"]),
33-
.testTarget(name: "TensorUtilsTests", dependencies: ["TensorUtils", "Models", "Hub"], resources: [.process("Resources")]),
34-
.testTarget(name: "NormalizerTests", dependencies: ["Tokenizers", "Hub"]),
35-
.testTarget(name: "PostProcessorTests", dependencies: ["Tokenizers", "Hub"]),
3622
]
3723
)

Sources/Generation/Generation.swift

Lines changed: 0 additions & 109 deletions
This file was deleted.

Sources/Generation/GenerationConfig.swift

Lines changed: 0 additions & 58 deletions
This file was deleted.

Sources/Hub/Downloader.swift

Lines changed: 0 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -7,128 +7,6 @@
77
//
88

99
import Foundation
10-
import Combine
11-
12-
class Downloader: NSObject, ObservableObject {
13-
private(set) var destination: URL
14-
15-
enum DownloadState {
16-
case notStarted
17-
case downloading(Double)
18-
case completed(URL)
19-
case failed(Error)
20-
}
21-
22-
enum DownloadError: Error {
23-
case invalidDownloadLocation
24-
case unexpectedError
25-
}
26-
27-
private(set) lazy var downloadState: CurrentValueSubject<DownloadState, Never> = CurrentValueSubject(.notStarted)
28-
private var stateSubscriber: Cancellable?
29-
30-
private var urlSession: URLSession? = nil
31-
32-
init(from url: URL, to destination: URL, using authToken: String? = nil, inBackground: Bool = false) {
33-
self.destination = destination
34-
super.init()
35-
let sessionIdentifier = "swift-transformers.hub.downloader"
36-
37-
var config = URLSessionConfiguration.default
38-
if inBackground {
39-
config = URLSessionConfiguration.background(withIdentifier: sessionIdentifier)
40-
config.isDiscretionary = false
41-
config.sessionSendsLaunchEvents = true
42-
}
43-
44-
self.urlSession = URLSession(configuration: config, delegate: self, delegateQueue: nil)
45-
46-
setupDownload(from: url, with: authToken)
47-
}
48-
49-
private func setupDownload(from url: URL, with authToken: String?) {
50-
downloadState.value = .downloading(0)
51-
urlSession?.getAllTasks { tasks in
52-
// If there's an existing pending background task with the same URL, let it proceed.
53-
if let existing = tasks.filter({ $0.originalRequest?.url == url }).first {
54-
switch existing.state {
55-
case .running:
56-
// print("Already downloading \(url)")
57-
return
58-
case .suspended:
59-
// print("Resuming suspended download task for \(url)")
60-
existing.resume()
61-
return
62-
case .canceling:
63-
// print("Starting new download task for \(url), previous was canceling")
64-
break
65-
case .completed:
66-
// print("Starting new download task for \(url), previous is complete but the file is no longer present (I think it's cached)")
67-
break
68-
@unknown default:
69-
// print("Unknown state for running task; cancelling and creating a new one")
70-
existing.cancel()
71-
}
72-
}
73-
var request = URLRequest(url: url)
74-
if let authToken = authToken {
75-
request.setValue("Bearer \(authToken)", forHTTPHeaderField: "Authorization")
76-
}
77-
78-
self.urlSession?.downloadTask(with: request).resume()
79-
}
80-
}
81-
82-
@discardableResult
83-
func waitUntilDone() throws -> URL {
84-
// It's either this, or stream the bytes ourselves (add to a buffer, save to disk, etc; boring and finicky)
85-
let semaphore = DispatchSemaphore(value: 0)
86-
stateSubscriber = downloadState.sink { state in
87-
switch state {
88-
case .completed: semaphore.signal()
89-
case .failed: semaphore.signal()
90-
default: break
91-
}
92-
}
93-
semaphore.wait()
94-
95-
switch downloadState.value {
96-
case .completed(let url): return url
97-
case .failed(let error): throw error
98-
default: throw DownloadError.unexpectedError
99-
}
100-
}
101-
102-
func cancel() {
103-
urlSession?.invalidateAndCancel()
104-
}
105-
}
106-
107-
extension Downloader: URLSessionDownloadDelegate {
108-
func urlSession(_: URLSession, downloadTask: URLSessionDownloadTask, didWriteData _: Int64, totalBytesWritten: Int64, totalBytesExpectedToWrite: Int64) {
109-
downloadState.value = .downloading(Double(totalBytesWritten) / Double(totalBytesExpectedToWrite))
110-
}
111-
112-
func urlSession(_: URLSession, downloadTask _: URLSessionDownloadTask, didFinishDownloadingTo location: URL) {
113-
do {
114-
// If the downloaded file already exists on the filesystem, overwrite it
115-
try FileManager.default.moveDownloadedFile(from: location, to: self.destination)
116-
downloadState.value = .completed(destination)
117-
} catch {
118-
downloadState.value = .failed(error)
119-
}
120-
}
121-
122-
func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
123-
if let error = error {
124-
downloadState.value = .failed(error)
125-
// } else if let response = task.response as? HTTPURLResponse {
126-
// print("HTTP response status code: \(response.statusCode)")
127-
// let headers = response.allHeaderFields
128-
// print("HTTP response headers: \(headers)")
129-
}
130-
}
131-
}
13210

13311
extension FileManager {
13412
func moveDownloadedFile(from srcURL: URL, to dstURL: URL) throws {

Sources/Hub/Hub.swift

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -112,15 +112,6 @@ public class LanguageModelConfigurationFromHub {
112112

113113
private var configPromise: Task<Configurations, Error>? = nil
114114

115-
public init(
116-
modelName: String,
117-
hubApi: HubApi = .shared
118-
) {
119-
self.configPromise = Task.init {
120-
return try await self.loadConfig(modelName: modelName, hubApi: hubApi)
121-
}
122-
}
123-
124115
public init(
125116
modelFolder: URL,
126117
hubApi: HubApi = .shared
@@ -173,17 +164,6 @@ public class LanguageModelConfigurationFromHub {
173164
}
174165
}
175166

176-
func loadConfig(
177-
modelName: String,
178-
hubApi: HubApi = .shared
179-
) async throws -> Configurations {
180-
let filesToDownload = ["config.json", "tokenizer_config.json", "tokenizer.json"]
181-
let repo = Hub.Repo(id: modelName)
182-
let downloadedModelFolder = try await hubApi.snapshot(from: repo, matching: filesToDownload)
183-
184-
return try await loadConfig(modelFolder: downloadedModelFolder, hubApi: hubApi)
185-
}
186-
187167
func loadConfig(
188168
modelFolder: URL,
189169
hubApi: HubApi = .shared

0 commit comments

Comments
 (0)