Skip to content

Commit 137e701

Browse files
tayloraswifttayloraswift
committed
remove index-wide rebuild implementation, and replace it with a per-edition uplink endpoint
1 parent d6e5a7c commit 137e701

File tree

10 files changed

+206
-185
lines changed

10 files changed

+206
-185
lines changed

‎Sources/UnidocDB/Packages/UnidocDatabase.Graphs.swift

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -71,28 +71,7 @@ extension UnidocDatabase.Graphs
7171
}
7272
extension UnidocDatabase.Graphs
7373
{
74-
func list(package:Int32? = nil,
75-
with session:Mongo.Session,
76-
_ yield:(Snapshot) async throws -> ()) async throws
77-
{
78-
try await self.list(package: package, version: nil, with: session, yield)
79-
}
80-
81-
func list(edition:Unidoc.Zone,
82-
with session:Mongo.Session,
83-
_ yield:(Snapshot) async throws -> ()) async throws
84-
{
85-
try await self.list(
86-
package: edition.package,
87-
version: edition.version,
88-
with: session,
89-
yield)
90-
}
91-
92-
private
93-
func list(
94-
package:Int32?,
95-
version:Int32?,
74+
func list(filter:(package:Int32, version:Int32?)? = nil,
9675
with session:Mongo.Session,
9776
_ yield:(Snapshot) async throws -> ()) async throws
9877
{
@@ -104,14 +83,14 @@ extension UnidocDatabase.Graphs
10483
{
10584
$0[.match] = .init
10685
{
107-
$0[Snapshot[.package]] = package
108-
$0[Snapshot[.version]] = version
86+
$0[Snapshot[.package]] = filter?.package
87+
$0[Snapshot[.version]] = filter?.version
10988
}
11089
}
11190
},
11291
stride: 1)
11392
{
114-
if case _? = package
93+
if case _? = filter?.package
11594
{
11695
$0[.hint] = .init
11796
{

‎Sources/UnidocDB/Packages/UnidocDatabase.swift

Lines changed: 127 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ extension UnidocDatabase:DatabaseModel
7272
try await self.siteMaps.setup(with: session)
7373
}
7474
}
75+
76+
@available(*, unavailable, message: "unused")
7577
extension UnidocDatabase
7678
{
7779
private
@@ -84,39 +86,96 @@ extension UnidocDatabase
8486
try await self.names.replace(with: session)
8587
try await self.siteMaps.replace(with: session)
8688
}
87-
}
88-
extension UnidocDatabase
89-
{
90-
// TODO: we need to get out of the habit of performing database-wide rebuilds;
91-
// this procedure should be deprecated!
92-
public
93-
func _rebuild(with session:__shared Mongo.Session) async throws -> Int
89+
90+
func _editions(of package:PackageIdentifier,
91+
with session:Mongo.Session) async throws -> [PackageEdition]
9492
{
95-
// TODO: we need to implement some kind of locking mechanism to prevent
96-
// race conditions between rebuilds and pushes. This cannot be done with
97-
// MongoDB transactions because deleting a very large number of records
98-
// overflows the transaction cache.
99-
try await self._clear(with: session)
93+
try await session.run(
94+
command: Mongo.Aggregate<Mongo.Cursor<PackageEdition>>.init(Packages.name,
95+
pipeline: .init
96+
{
97+
$0.stage
98+
{
99+
$0[.match] = .init
100+
{
101+
$0[PackageRecord[.id]] = package
102+
}
103+
}
100104

101-
print("cleared all unidoc volumes...")
105+
let editions:Mongo.KeyPath = "editions"
102106

103-
// var origins:[Int32: Volume.Origin?] = [:]
104-
var count:Int = 0
107+
$0.stage
108+
{
109+
$0[.lookup] = .init
110+
{
111+
$0[.from] = Editions.name
112+
$0[.localField] = PackageRecord[.cell]
113+
$0[.foreignField] = PackageEdition[.package]
114+
$0[.as] = editions
115+
}
116+
}
105117

106-
try await self.graphs.list(with: session)
107-
{
108-
(snapshot:Snapshot) in
118+
$0.stage
119+
{
120+
$0[.lookup] = .init
121+
{
122+
let cell:Mongo.Variable<Int32> = "cell"
109123

110-
let volume:Volume = try await self.link(snapshot, with: session)
124+
$0[.from] = Graphs.name
125+
$0[.let] = .init
126+
{
127+
$0[let: cell] = PackageRecord[.cell]
128+
}
129+
$0[.pipeline] = .init
130+
{
131+
$0.stage
132+
{
133+
$0[.match] = .init
134+
{
135+
$0[.expr] = .expr
136+
{
137+
$0[.eq] = (Snapshot[.package], cell)
138+
}
139+
}
140+
}
111141

112-
try await self.publish(volume, with: session)
142+
$0.stage
143+
{
144+
$0[.sort] = .init
145+
{
146+
$0[Snapshot[.version]] = (-)
147+
}
148+
}
113149

114-
count += 1
150+
$0.stage
151+
{
152+
$0[.limit] = 1
153+
}
115154

116-
print("regenerated records for snapshot: \(snapshot.id)")
117-
}
155+
$0.stage
156+
{
157+
$0[.replaceWith] = Snapshot[.metadata]
158+
}
159+
}
160+
$0[.as] = "recent"
161+
}
162+
}
163+
164+
$0.stage
165+
{
166+
$0[.unwind] = editions
167+
}
118168

119-
return count
169+
$0.stage
170+
{
171+
$0[.replaceWith] = editions
172+
}
173+
},
174+
stride: 1),
175+
against: self.id)
176+
{
177+
try await $0.reduce(into: [], += )
178+
}
120179
}
121180
}
122181

@@ -209,135 +268,72 @@ extension UnidocDatabase
209268
repo: placement.repo)
210269
}
211270
}
212-
extension UnidocDatabase
213-
{
214-
@available(*, unavailable, message: "unused")
215-
func _editions(of package:PackageIdentifier,
216-
with session:Mongo.Session) async throws -> [PackageEdition]
217-
{
218-
try await session.run(
219-
command: Mongo.Aggregate<Mongo.Cursor<PackageEdition>>.init(Packages.name,
220-
pipeline: .init
221-
{
222-
$0.stage
223-
{
224-
$0[.match] = .init
225-
{
226-
$0[PackageRecord[.id]] = package
227-
}
228-
}
229-
230-
let editions:Mongo.KeyPath = "editions"
231-
232-
$0.stage
233-
{
234-
$0[.lookup] = .init
235-
{
236-
$0[.from] = Editions.name
237-
$0[.localField] = PackageRecord[.cell]
238-
$0[.foreignField] = PackageEdition[.package]
239-
$0[.as] = editions
240-
}
241-
}
242-
243-
$0.stage
244-
{
245-
$0[.lookup] = .init
246-
{
247-
let cell:Mongo.Variable<Int32> = "cell"
248-
249-
$0[.from] = Graphs.name
250-
$0[.let] = .init
251-
{
252-
$0[let: cell] = PackageRecord[.cell]
253-
}
254-
$0[.pipeline] = .init
255-
{
256-
$0.stage
257-
{
258-
$0[.match] = .init
259-
{
260-
$0[.expr] = .expr
261-
{
262-
$0[.eq] = (Snapshot[.package], cell)
263-
}
264-
}
265-
}
266-
267-
$0.stage
268-
{
269-
$0[.sort] = .init
270-
{
271-
$0[Snapshot[.version]] = (-)
272-
}
273-
}
274-
275-
$0.stage
276-
{
277-
$0[.limit] = 1
278-
}
279-
280-
$0.stage
281-
{
282-
$0[.replaceWith] = Snapshot[.metadata]
283-
}
284-
}
285-
$0[.as] = "recent"
286-
}
287-
}
288-
289-
$0.stage
290-
{
291-
$0[.unwind] = editions
292-
}
293-
294-
$0.stage
295-
{
296-
$0[.replaceWith] = editions
297-
}
298-
},
299-
stride: 1),
300-
against: self.id)
301-
{
302-
try await $0.reduce(into: [], += )
303-
}
304-
}
305-
}
306271

307272
extension UnidocDatabase
308273
{
309274
public
310-
func publish(linking docs:__owned SymbolGraphArchive,
275+
func publish(_ docs:__owned SymbolGraphArchive,
311276
with session:Mongo.Session) async throws -> SnapshotReceipt
312277
{
313278
let receipt:SnapshotReceipt = try await self.store(docs: docs, with: session)
279+
314280
let volume:Volume = try await self.link(.init(
315281
package: receipt.package,
316282
version: receipt.version,
317283
metadata: docs.metadata,
318284
graph: docs.graph),
319285
with: session)
320286

321-
if case .update = receipt.type
322-
{
323-
try await self.search.delete(volume.id, with: session)
287+
_ = consume docs
324288

325-
try await self.vertices.clear(receipt.edition, with: session)
326-
try await self.groups.clear(receipt.edition, with: session)
327-
try await self.trees.clear(receipt.edition, with: session)
289+
try await self.fill(volume: consume volume,
290+
clear: receipt.type == .update,
291+
with: session)
292+
293+
return receipt
294+
}
328295

329-
try await self.names.delete(receipt.edition, with: session)
296+
public
297+
func uplink(
298+
package:Int32,
299+
version:Int32?,
300+
with session:Mongo.Session) async throws -> Int
301+
{
302+
var uplinked:Int = 0
303+
304+
try await self.graphs.list(
305+
filter: (package: package, version: version),
306+
with: session)
307+
{
308+
try await self.fill(
309+
volume: try await self.link($0, with: session),
310+
clear: true,
311+
with: session)
312+
313+
uplinked += 1
330314
}
331315

332-
try await self.publish(volume, with: session)
333-
return receipt
316+
return uplinked
334317
}
335318
}
336319
extension UnidocDatabase
337320
{
338321
private
339-
func publish(_ volume:__owned Volume, with session:__shared Mongo.Session) async throws
322+
func fill(volume:__owned Volume,
323+
clear:Bool = true,
324+
with session:Mongo.Session) async throws
340325
{
326+
if clear
327+
{
328+
try await self.search.delete(volume.id, with: session)
329+
330+
try await self.vertices.clear(volume.edition, with: session)
331+
try await self.groups.clear(volume.edition, with: session)
332+
try await self.trees.clear(volume.edition, with: session)
333+
334+
try await self.names.delete(volume.edition, with: session)
335+
}
336+
341337
let (index, trees):(SearchIndex<VolumeIdentifier>, [Volume.TypeTree]) = volume.indexes()
342338

343339
try await self.vertices.insert(some: volume.vertices, with: session)
@@ -416,8 +412,8 @@ extension UnidocDatabase
416412
extension UnidocDatabase
417413
{
418414
public
419-
func siteMap(package:__owned PackageIdentifier,
420-
with session:__shared Mongo.Session) async throws -> Volume.SiteMap<PackageIdentifier>?
415+
func siteMap(package:consuming PackageIdentifier,
416+
with session:Mongo.Session) async throws -> Volume.SiteMap<PackageIdentifier>?
421417
{
422418
try await self.siteMaps.find(by: package, with: session)
423419
}

‎Sources/UnidocPages/API/Site.API.Post.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ extension Site.API
44
enum Post:String
55
{
66
case index
7+
case uplink
78
}
89
}
910
extension Site.API.Post:StaticAPI

0 commit comments

Comments
 (0)