Skip to content

Commit e73c6cb

Browse files
tayloraswifttayloraswift
committed
generalize the HTTP2Client, and sketch out symbol graph uploading logic in UnidocBuilder
1 parent 9f46326 commit e73c6cb

File tree

8 files changed

+214
-36
lines changed

8 files changed

+214
-36
lines changed

‎Package.swift

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,8 +458,9 @@ let package:Package = .init(
458458
.executableTarget(name: "UnidocBuild", dependencies:
459459
[
460460
.target(name: "HTTPClient"),
461-
.target(name: "UnidocAutomation"),
462461
.target(name: "SymbolGraphBuilder"),
462+
.target(name: "UnidocAutomation"),
463+
.target(name: "UnidocLinker"),
463464
]),
464465

465466
.executableTarget(name: "UnidocServer", dependencies:

‎Sources/HTTPClient/ClientInterfaceHandler.swift

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ extension ClientInterfaceHandler:ChannelOutboundHandler
4141
typealias OutboundIn =
4242
(
4343
owner:AsyncThrowingStream<HTTP2Client.Facet, any Error>.Continuation,
44-
batch:[HPACKHeaders]
44+
batch:[HTTP2Client.Request]
4545
)
4646

4747
func errorCaught(context:ChannelHandlerContext, error:any Error)
@@ -53,14 +53,14 @@ extension ClientInterfaceHandler:ChannelOutboundHandler
5353
func write(context:ChannelHandlerContext, data:NIOAny, promise:EventLoopPromise<Void>?)
5454
{
5555
let owner:AsyncThrowingStream<HTTP2Client.Facet, any Error>.Continuation
56-
let batch:[HPACKHeaders]
56+
let batch:[HTTP2Client.Request]
5757

5858
(owner, batch) = self.unwrapOutboundIn(data)
5959

6060
self.owner?.finish()
6161
self.owner = owner
6262

63-
for request:HPACKHeaders in batch
63+
for request:HTTP2Client.Request in batch
6464
{
6565
self.multiplexer.createStreamChannel
6666
{
@@ -71,11 +71,27 @@ extension ClientInterfaceHandler:ChannelOutboundHandler
7171
switch $0
7272
{
7373
case .success(let stream):
74-
stream.writeAndFlush(HTTP2Frame.FramePayload.headers(
75-
HTTP2Frame.FramePayload.Headers.init(
76-
headers: request,
77-
endStream: true)),
78-
promise: nil)
74+
if let body:ByteBuffer = request.body
75+
{
76+
stream.write(HTTP2Frame.FramePayload.headers(
77+
HTTP2Frame.FramePayload.Headers.init(
78+
headers: request.headers,
79+
endStream: false)),
80+
promise: nil)
81+
stream.writeAndFlush(HTTP2Frame.FramePayload.data(
82+
HTTP2Frame.FramePayload.Data.init(
83+
data: .byteBuffer(body),
84+
endStream: true)),
85+
promise: nil)
86+
}
87+
else
88+
{
89+
stream.writeAndFlush(HTTP2Frame.FramePayload.headers(
90+
HTTP2Frame.FramePayload.Headers.init(
91+
headers: request.headers,
92+
endStream: true)),
93+
promise: nil)
94+
}
7995

8096
case .failure(let error):
8197
owner.finish(throwing: error)

‎Sources/HTTPClient/HTTP2Client.Connection.swift

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,21 +25,46 @@ extension HTTP2Client.Connection:Sendable
2525
{
2626
}
2727
extension HTTP2Client.Connection
28+
{
29+
@inlinable public
30+
func buffer(string:Substring) -> ByteBuffer
31+
{
32+
self.channel.allocator.buffer(substring: string)
33+
}
34+
35+
@inlinable public
36+
func buffer(string:String) -> ByteBuffer
37+
{
38+
self.channel.allocator.buffer(string: string)
39+
}
40+
41+
@inlinable public
42+
func buffer(bytes:[UInt8]) -> ByteBuffer
43+
{
44+
self.channel.allocator.buffer(bytes: bytes)
45+
}
46+
}
47+
extension HTTP2Client.Connection
2848
{
2949
public
3050
func fetch(_ request:__owned HPACKHeaders) async throws -> HTTP2Client.Facet
51+
{
52+
try await self.fetch(.init(headers: request, body: nil))
53+
}
54+
public
55+
func fetch(_ request:__owned HTTP2Client.Request) async throws -> HTTP2Client.Facet
3156
{
3257
try await self.fetch(reducing: [request], into: .init()) { $0 = $1 }
3358
}
3459

3560
public
36-
func fetch(_ batch:__owned [HPACKHeaders]) async throws -> [HTTP2Client.Facet]
61+
func fetch(_ batch:__owned [HTTP2Client.Request]) async throws -> [HTTP2Client.Facet]
3762
{
3863
try await self.fetch(reducing: batch, into: []) { $0.append($1) }
3964
}
4065

4166
@inlinable public
42-
func fetch<Response>(reducing batch:__owned [HPACKHeaders],
67+
func fetch<Response>(reducing batch:__owned [HTTP2Client.Request],
4368
into initial:__owned Response,
4469
with combine:(inout Response, HTTP2Client.Facet) throws -> ()) async throws -> Response
4570
{

‎Sources/HTTPClient/HTTP2Client.Facet.swift

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,12 @@ extension HTTP2Client.Facet
5757
self.buffers.append(buffer)
5858
return frame.endStream
5959
}
60-
case _:
60+
61+
case .rstStream, .pushPromise, .goAway:
6162
break
63+
64+
case _:
65+
return false
6266
}
6367

6468
throw HTTP2Client.UnexpectedFrameError.init(payload)

‎Sources/HTTPClient/HTTP2Client.Request.swift

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import NIOCore
2+
import NIOHPACK
3+
4+
extension HTTP2Client
5+
{
6+
@frozen public
7+
struct Request:Sendable
8+
{
9+
public
10+
var headers:HPACKHeaders
11+
public
12+
var body:ByteBuffer?
13+
14+
@inlinable public
15+
init(headers:HPACKHeaders = [:], body:ByteBuffer?)
16+
{
17+
self.headers = headers
18+
self.body = body
19+
}
20+
}
21+
}

‎Sources/HTTPClient/HTTP2Client.swift

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ extension HTTP2Client
7979
{
8080
try await self.connect { try await $0.fetch(request) }
8181
}
82+
public
83+
func fetch(_ request:__owned HTTP2Client.Request) async throws -> Facet
84+
{
85+
try await self.connect { try await $0.fetch(request) }
86+
}
8287
}
8388
extension HTTP2Client
8489
{

‎Sources/UnidocBuild/Main.swift

Lines changed: 115 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
1+
import BSON
12
import HTTPClient
23
import ModuleGraphs
34
import NIOCore
45
import NIOPosix
56
import NIOSSL
7+
import SymbolGraphBuilder
8+
import SymbolGraphs
69
import UnidocAutomation
10+
import UnidocLinker
711

812
@main
913
enum Main
@@ -26,14 +30,50 @@ enum Main
2630

2731
let swiftinit:SwiftinitClient = .init(http2: http2, cookie: options.cookie)
2832

29-
let status:PackageBuildStatus = try await swiftinit.get(
30-
from: "/api/build/\(options.package)")
33+
try await swiftinit.connect
34+
{
35+
let package:PackageBuildStatus = try await $0.get(
36+
from: "/api/build/\(options.package)")
37+
38+
let toolchain:Toolchain = try await .detect()
39+
let workspace:Workspace = try await .create(at: ".swiftinit")
40+
41+
let edition:PackageBuildStatus.Edition
42+
43+
if package.release.graphs == 0
44+
{
45+
edition = package.release
46+
}
47+
else if
48+
let prerelease:PackageBuildStatus.Edition = package.prerelease,
49+
prerelease.graphs == 0
50+
{
51+
edition = prerelease
52+
}
53+
else
54+
{
55+
print("No new documentation to build")
56+
return
57+
}
58+
59+
let build:PackageBuild = try await .remote(
60+
package: options.package,
61+
from: package.repo,
62+
at: edition.tag,
63+
in: workspace,
64+
clean: true)
3165

32-
let builder:Massbuilder = try await .init()
66+
let snapshot:Snapshot = .init(
67+
package: package.coordinate,
68+
version: edition.coordinate,
69+
archive: try await toolchain.generateDocs(for: build, pretty: options.pretty))
3370

34-
try await builder.build(options.package,
35-
repository: status.repo,
36-
at: status.release.tag)
71+
let bson:BSON.Document = .init(encoding: consume snapshot)
72+
73+
try await $0.put(bson: bson, to: "/api/symbolgraph")
74+
75+
print("Successfully uploaded symbol graph (tag: \(edition.tag))")
76+
}
3777
}
3878
}
3979

@@ -44,13 +84,15 @@ extension Main
4484
var package:PackageIdentifier
4585
var cookie:String
4686
var remote:String
87+
var pretty:Bool
4788

4889
private
4990
init(package:PackageIdentifier)
5091
{
5192
self.package = package
5293
self.cookie = ""
5394
self.remote = "swiftinit.org"
95+
self.pretty = false
5496
}
5597
}
5698
}
@@ -94,6 +136,9 @@ extension Main.Options
94136

95137
options.remote = remote
96138

139+
case "--pretty", "-p":
140+
options.pretty = true
141+
97142
case let option:
98143
fatalError("Unknown option '\(option)'")
99144
}
@@ -144,7 +189,9 @@ extension SwiftinitClient
144189
}
145190

146191
import JSON
192+
import Media
147193
import NIOHPACK
194+
import URI
148195

149196
extension SwiftinitClient
150197
{
@@ -169,40 +216,84 @@ extension SwiftinitClient
169216
}
170217
extension SwiftinitClient.Connection
171218
{
219+
@inlinable internal
220+
func headers(_ method:String, _ endpoint:String) -> HPACKHeaders
221+
{
222+
[
223+
":method": method,
224+
":scheme": "https",
225+
":authority": self.remote,
226+
":path": endpoint,
227+
228+
"user-agent": "UnidocBuild",
229+
"accept": "application/json",
230+
"cookie": "__Host-session=\(self.cookie)",
231+
]
232+
}
233+
}
234+
extension SwiftinitClient.Connection
235+
{
236+
@discardableResult
237+
@inlinable public
238+
func post(urlencoded:consuming String, to endpoint:String) async throws -> [ByteBuffer]
239+
{
240+
try await self.fetch(endpoint, method: "POST",
241+
body: self.http2.buffer(string: urlencoded),
242+
type: .application(.x_www_form_urlencoded))
243+
}
244+
245+
@discardableResult
246+
@inlinable public
247+
func put(bson:consuming BSON.Document, to endpoint:String) async throws -> [ByteBuffer]
248+
{
249+
try await self.fetch(endpoint, method: "PUT",
250+
body: self.http2.buffer(bytes: (consume bson).bytes),
251+
type: .application(.bson))
252+
}
253+
172254
@inlinable public
173255
func get<Response>(_:Response.Type = Response.self,
174256
from endpoint:String) async throws -> Response where Response:JSONDecodable
257+
{
258+
var json:JSON = .init(utf8: [])
259+
260+
for buffer:ByteBuffer in try await self.fetch(endpoint, method: "GET")
261+
{
262+
json.utf8 += buffer.readableBytesView
263+
}
264+
265+
return try json.decode()
266+
}
267+
}
268+
extension SwiftinitClient.Connection
269+
{
270+
@inlinable internal
271+
func fetch(_ endpoint:String,
272+
method:String,
273+
body:ByteBuffer? = nil,
274+
type:MediaType? = nil) async throws -> [ByteBuffer]
175275
{
176276
var endpoint:String = endpoint
177277
var status:UInt? = nil
178278

179279
following:
180280
for _:Int in 0 ... 1
181281
{
182-
let request:HPACKHeaders =
183-
[
184-
":method": "GET",
185-
":scheme": "https",
186-
":authority": self.remote,
187-
":path": endpoint,
188-
189-
"user-agent": "UnidocBuild",
190-
"accept": "application/json",
191-
"cookie": "__Host-session=\(self.cookie)",
192-
]
282+
var headers:HPACKHeaders = self.headers(method, endpoint)
283+
if let type:MediaType
284+
{
285+
headers.add(name: "content-type", value: "\(type)")
286+
}
193287

194-
let response:HTTP2Client.Facet = try await self.http2.fetch(request)
288+
let response:HTTP2Client.Facet = try await self.http2.fetch(.init(
289+
headers: headers,
290+
body: body))
195291

196292
switch response.status
197293
{
198294
case 200?:
199-
var json:JSON = .init(utf8: [])
200-
for buffer:ByteBuffer in response.buffers
201-
{
202-
json.utf8 += buffer.readableBytesView
203-
}
295+
return response.buffers
204296

205-
return try json.decode()
206297
case 301?:
207298
if let location:String = response.headers?["location"].first
208299
{

‎Sources/UnidocLinker/Snapshot.swift

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,21 @@ struct Snapshot:Equatable, Sendable
3131
self.graph = graph
3232
}
3333
}
34+
extension Snapshot
35+
{
36+
@inlinable public
37+
init(
38+
package:Int32,
39+
version:Int32,
40+
archive:SymbolGraphArchive)
41+
{
42+
self.init(
43+
package: package,
44+
version: version,
45+
metadata: archive.metadata,
46+
graph: archive.graph)
47+
}
48+
}
3449
extension Snapshot:Identifiable
3550
{
3651
@inlinable public

0 commit comments

Comments
 (0)