Skip to content

Commit 297a3f8

Browse files
tayloraswifttayloraswift
committed
implement a way to gradually uplink all symbol graphs in the database
1 parent 4fdf961 commit 297a3f8

20 files changed

+291
-44
lines changed

‎Assets/css/Main.css

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Assets/css/Main.css.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Sources/Swiftinit/Tree/Swiftinit.API.Post.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ extension Swiftinit.API
66
case alignPackage = "align-package"
77
case indexRepo = "index-repo"
88
case indexRepoTag = "index-repo-tag"
9+
case uplinkAll = "uplink-all"
910
case uplink
1011
case unlink
1112
}

‎Sources/UnidocDB/Packages/UnidocDatabase.Packages.AlignRealm.swift

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,78 @@ import MongoQL
33
import Unidoc
44
import UnidocRecords
55

6+
extension UnidocDatabase.Snapshots
7+
{
8+
enum QueueUplink
9+
{
10+
case all
11+
}
12+
}
13+
extension UnidocDatabase.Snapshots.QueueUplink:Mongo.UpdateQuery
14+
{
15+
typealias Target = UnidocDatabase.Snapshots
16+
typealias Effect = Mongo.Many
17+
18+
var ordered:Bool { false }
19+
20+
func build(updates:inout Mongo.UpdateEncoder<Mongo.Many>)
21+
{
22+
updates
23+
{
24+
$0[.multi] = true
25+
26+
switch self
27+
{
28+
case .all:
29+
$0[.q] = [:]
30+
}
31+
32+
$0[.u] = .init
33+
{
34+
$0[.set] = .init
35+
{
36+
$0[Unidoc.Snapshot[.uplinking]] = true
37+
}
38+
}
39+
}
40+
}
41+
}
42+
extension UnidocDatabase.Snapshots
43+
{
44+
enum ClearUplink
45+
{
46+
case one(Unidoc.Edition)
47+
}
48+
}
49+
extension UnidocDatabase.Snapshots.ClearUplink:Mongo.UpdateQuery
50+
{
51+
typealias Target = UnidocDatabase.Snapshots
52+
typealias Effect = Mongo.One
53+
54+
var ordered:Bool { false }
55+
56+
func build(updates:inout Mongo.UpdateEncoder<Mongo.One>)
57+
{
58+
updates
59+
{
60+
61+
switch self
62+
{
63+
case .one(let edition):
64+
$0[.q] = .init { $0[Unidoc.Snapshot[.id]] = edition }
65+
}
66+
67+
$0[.u] = .init
68+
{
69+
$0[.unset] = .init
70+
{
71+
$0[Unidoc.Snapshot[.uplinking]] = ()
72+
}
73+
}
74+
}
75+
}
76+
}
77+
678
extension UnidocDatabase.Packages
779
{
880
enum AlignRealm

‎Sources/UnidocDB/Packages/UnidocDatabase.Snapshots.swift

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,17 @@ extension UnidocDatabase.Snapshots
3434
{
3535
$0[Unidoc.Snapshot[.swift]] = .init { $0[.exists] = true }
3636
}
37+
38+
public static
39+
let indexUplinking:Mongo.CollectionIndex = .init("Uplinking",
40+
unique: false)
41+
{
42+
$0[Unidoc.Snapshot[.uplinking]] = (+)
43+
}
44+
where:
45+
{
46+
$0[Unidoc.Snapshot[.uplinking]] = .init { $0[.exists] = true }
47+
}
3748
}
3849
extension UnidocDatabase.Snapshots:Mongo.CollectionModel
3950
{
@@ -44,7 +55,13 @@ extension UnidocDatabase.Snapshots:Mongo.CollectionModel
4455
var name:Mongo.Collection { "Snapshots" }
4556

4657
@inlinable public static
47-
var indexes:[Mongo.CollectionIndex] { [ Self.indexSwiftReleases ] }
58+
var indexes:[Mongo.CollectionIndex]
59+
{
60+
[
61+
Self.indexSwiftReleases,
62+
Self.indexUplinking,
63+
]
64+
}
4865
}
4966
extension UnidocDatabase.Snapshots
5067
{
@@ -127,3 +144,29 @@ extension UnidocDatabase.Snapshots
127144
}
128145
}
129146
}
147+
extension UnidocDatabase.Snapshots
148+
{
149+
func linkable(_ limit:Int,
150+
with session:Mongo.Session) async throws -> [Unidoc.Edition]?
151+
{
152+
let editions:[Mongo.IdentityView<Unidoc.Edition>] = try await session.run(
153+
command: Mongo.Find<Mongo.SingleBatch<Mongo.IdentityView<Unidoc.Edition>>>.init(
154+
Self.name,
155+
limit: limit)
156+
{
157+
$0[.filter] = .init
158+
{
159+
$0[Unidoc.Snapshot[.uplinking]] = true
160+
}
161+
$0[.projection] = .init
162+
{
163+
$0[Unidoc.Snapshot[.id]] = true
164+
}
165+
166+
$0[.hint] = Self.indexUplinking.id
167+
},
168+
against: self.database)
169+
170+
return editions.isEmpty ? nil : editions.map(\.id)
171+
}
172+
}

‎Sources/UnidocDB/UnidocDatabase.swift

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,51 @@ extension UnidocDatabase
570570
}
571571
extension UnidocDatabase
572572
{
573+
public
574+
func rebuild(with session:Mongo.Session) async throws
575+
{
576+
try await self.execute(update: Snapshots.QueueUplink.all, with: session)
577+
578+
while
579+
let editions:[Unidoc.Edition] = try await self.snapshots.linkable(8,
580+
with: session)
581+
{
582+
for edition:Unidoc.Edition in editions
583+
{
584+
async
585+
let cooldown:Void = Task.sleep(for: .seconds(10))
586+
587+
guard
588+
let status:Unidoc.UplinkStatus = try await self.uplink(edition, with: session)
589+
else
590+
{
591+
print("failed to uplink \(edition): volume not found")
592+
continue
593+
}
594+
595+
try await self.execute(
596+
update: Snapshots.ClearUplink.one(edition),
597+
with: session)
598+
599+
if let sitemap:Unidoc.SitemapDelta = status.sitemap
600+
{
601+
print("""
602+
successfully uplinked \(edition): \
603+
sitemap gained \(sitemap.additions) pages, \
604+
lost \(sitemap.deletions.count) pages
605+
""")
606+
}
607+
else
608+
{
609+
print("successfully uplinked \(edition)")
610+
}
611+
612+
613+
try await cooldown
614+
}
615+
}
616+
}
617+
573618
public
574619
func align(
575620
package:Unidoc.Package,

‎Sources/UnidocDB/Volumes/Unidoc.Metadata (ext).swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,6 @@ extension Unidoc.VolumeMetadata
1717
project[Unidoc.VolumeMetadata[.latest]] = true
1818
project[Unidoc.VolumeMetadata[.realm]] = true
1919
project[Unidoc.VolumeMetadata[.patch]] = true
20-
project[Unidoc.VolumeMetadata[.api]] = true
20+
project[Unidoc.VolumeMetadata[.abi]] = true
2121
}
2222
}

‎Sources/UnidocPages/Responses/Unidoc.EditionsPage.Row.swift

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,16 @@ extension Unidoc.EditionsPage.Row:HyperTextOutputStreamable
4848
{
4949
if let graph:Unidoc.EditionOutput.Graph = self.graph
5050
{
51-
$0[.span] { $0.class = "abi" } = "\(graph.abi)"
51+
$0[.span]
52+
{
53+
$0.class = graph.uplinking ? "abi uplinking" : "abi"
54+
$0.title = graph.uplinking ? """
55+
This symbol graph is currently queued for documentation generation.
56+
""" : nil
57+
} = "\(graph.abi)"
58+
5259
$0 += " "
60+
5361
$0[.span]
5462
{
5563
$0.class = "kb"

‎Sources/UnidocPages/Templates/Swiftinit.AdminPage.swift

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -162,30 +162,30 @@ extension Swiftinit.AdminPage:AdministrativePage
162162
}
163163

164164
// Non-destructive actions.
165-
// for (action, label):(Site.API.Post, String) in
166-
// [
167-
// (
168-
// .reloadAssets,
169-
// "Reload Assets"
170-
// ),
171-
// ]
172-
// {
173-
// main[.hr]
174-
175-
// main[.form]
176-
// {
177-
// $0.enctype = "\(MediaType.application(.x_www_form_urlencoded))"
178-
// $0.action = "\(Site.API[action])"
179-
// $0.method = "post"
180-
// }
181-
// content:
182-
// {
183-
// $0[.p]
184-
// {
185-
// $0[.button] { $0.type = "submit" } = label
186-
// }
187-
// }
188-
// }
165+
for (action, label):(Swiftinit.API.Post, String) in
166+
[
167+
(
168+
.uplinkAll,
169+
"Uplink all volumes"
170+
),
171+
]
172+
{
173+
main[.hr]
174+
175+
main[.form]
176+
{
177+
$0.enctype = "\(MediaType.application(.x_www_form_urlencoded))"
178+
$0.action = "\(Swiftinit.API[action])"
179+
$0.method = "post"
180+
}
181+
content:
182+
{
183+
$0[.p]
184+
{
185+
$0[.button] { $0.type = "submit" } = label
186+
}
187+
}
188+
}
189189

190190
// Destructive actions.
191191
for action:Action in

‎Sources/UnidocPages/UI/HTML.Sidebar.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ extension HTML.Sidebar:HyperTextOutputStreamable
8787
// The URI is only valid if the principal volume API version is at
8888
// least 1.0!
8989
if case .foreign = citizenship,
90-
self.volume.api < .v(1, 0)
90+
self.volume.abi < .v(1, 0)
9191
{
9292
$0[.span] = name
9393
}

0 commit comments

Comments
 (0)