Skip to content

Commit df2de3e

Browse files
tayloraswifttayloraswift
committed
sketch implementation of the GitHub repo telescope
1 parent f8d0222 commit df2de3e

24 files changed

+630
-353
lines changed

‎Sources/S3/AWS.AccessKey.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ extension AWS.AccessKey
9595
hash:SHA256) -> String
9696
{
9797
let yyyymmddThhmmssZ:String = timestamp.components.yyyymmddThhmmssZ
98-
let yyyymmdd:String = timestamp.components.yyyymmdd
98+
let yyyymmdd:String = timestamp.date.yyyymmdd
9999

100100
let headers:String = "date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"
101101
let request:String = """

‎Sources/S3Tests/Main.swift

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,9 @@ enum Main:TestMain, TestBattery
1717
bucket: .init(
1818
region: .us_east_1,
1919
name: "examplebucket"),
20-
date: .init(
21-
components: .init(
22-
year: 2013,
23-
month: 5,
24-
day: 24,
25-
hour: 0,
26-
minute: 0,
27-
second: 0),
28-
weekday: .friday),
20+
date: .init(weekday: .friday,
21+
date: .init(year: 2013, month: 5, day: 24),
22+
time: .init(hour: 0, minute: 0, second: 0)),
2923
path: "/test%24file.text")
3024

3125
tests.expect(computed ==? """

‎Sources/SwiftinitPages/Surfaces/Editions/Swiftinit.TagsPage.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ extension Swiftinit.TagsPage:Swiftinit.ApplicationPage
141141

142142
if let created:Timestamp.Components = .init(iso8601: repo.created)
143143
{
144+
let created:Timestamp.Date = created.date
145+
144146
$0[.dt] = "Created"
145147
$0[.dd] = "\(created.month(.en)) \(created.day), \(created.year)"
146148
}

‎Sources/SwiftinitServer/Endpoints/Interactive/Swiftinit.PackageIndexEndpoint.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ extension Swiftinit.PackageIndexEndpoint:RestrictedEndpoint
3232
return nil
3333
}
3434

35-
let response:GitHubPlugin.CrawlerResponse = try await github.api.connect
35+
let response:GitHubPlugin.RepoMonitorResponse = try await github.api.connect
3636
{
3737
try await $0.crawl(owner: self.owner,
3838
repo: self.repo,

‎Sources/SwiftinitServer/Endpoints/Interactive/Swiftinit.SitemapIndexEndpoint.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ extension Swiftinit.SitemapIndexEndpoint:PublicEndpoint
5353

5454
$0[.lastmod] = modified.timestamp.map
5555
{
56-
"\($0.components.year)-\($0.components.MM)-\($0.components.DD)"
56+
"\($0.date.year)-\($0.date.mm)-\($0.date.dd)"
5757
}
5858
}
5959
}

‎Sources/SwiftinitServer/Plugins/GitHub.RepoNode.swift

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import GitHubAPI
2+
import JSON
3+
4+
extension GitHub
5+
{
6+
/// A wrapper around a ``Repo`` that uses the GraphQL-flavored format.
7+
struct RepoNode
8+
{
9+
let repo:Repo
10+
11+
init(repo:Repo)
12+
{
13+
self.repo = repo
14+
}
15+
}
16+
}
17+
extension GitHub.RepoNode:JSONObjectDecodable
18+
{
19+
enum CodingKey:String, Sendable
20+
{
21+
case id
22+
case owner
23+
case name
24+
25+
case license
26+
enum License:String, Sendable
27+
{
28+
case id
29+
case name
30+
}
31+
32+
case topics
33+
enum Topics:String, Sendable
34+
{
35+
case nodes
36+
enum Node:String, Sendable
37+
{
38+
case topic
39+
enum Topic:String, Sendable
40+
{
41+
case name
42+
}
43+
}
44+
}
45+
46+
case master
47+
enum Master:String, Sendable
48+
{
49+
case name
50+
}
51+
52+
case watchers
53+
enum Watchers:String, Sendable
54+
{
55+
case count
56+
}
57+
58+
case forks
59+
case stars
60+
case size
61+
62+
case archived
63+
case disabled
64+
case fork
65+
66+
case homepage
67+
case about
68+
69+
case created
70+
case updated
71+
case pushed
72+
73+
// Not present in this type, but could be added as a table join.
74+
case refs
75+
enum Refs:String, Sendable
76+
{
77+
case nodes
78+
}
79+
}
80+
81+
init(json:JSON.ObjectDecoder<CodingKey>) throws
82+
{
83+
self.init(repo: .init(id: try json[.id].decode(),
84+
owner: try json[.owner].decode(),
85+
name: try json[.name].decode(),
86+
license: try json[.license].decode(as: JSON.ObjectDecoder<CodingKey.License>?.self)
87+
{
88+
guard
89+
let json:JSON.ObjectDecoder<CodingKey.License> = $0
90+
else
91+
{
92+
return nil
93+
}
94+
// The GraphQL API is slightly different from the REST API. The license
95+
// field is always present, but the license id is not. For consistency,
96+
// we consider the license to be nil if the id is nil.
97+
if let id:String = try json[.id]?.decode()
98+
{
99+
return .init(id: id, name: try json[.name].decode())
100+
}
101+
else
102+
{
103+
return nil
104+
}
105+
},
106+
topics: try json[.topics].decode(using: CodingKey.Topics.self)
107+
{
108+
try $0[.nodes].decode(as: JSON.Array.self)
109+
{
110+
try $0.map
111+
{
112+
try $0.decode(using: CodingKey.Topics.Node.self)
113+
{
114+
try $0[.topic].decode(using: CodingKey.Topics.Node.Topic.self)
115+
{
116+
try $0[.name].decode()
117+
}
118+
}
119+
}
120+
}
121+
},
122+
// This is actually nil if the repo is empty.
123+
// But we would never crawl an empty repo.
124+
master: try json[.master].decode(using: CodingKey.Master.self)
125+
{
126+
try $0[.name].decode()
127+
},
128+
watchers: try json[.watchers].decode(using: CodingKey.Watchers.self)
129+
{
130+
try $0[.count].decode()
131+
},
132+
forks: try json[.forks].decode(),
133+
stars: try json[.stars].decode(),
134+
size: try json[.size].decode(),
135+
archived: try json[.archived].decode(),
136+
disabled: try json[.disabled].decode(),
137+
fork: try json[.fork].decode(),
138+
homepage: try json[.homepage]?.decode(as: String.self) { $0.isEmpty ? nil : $0 },
139+
about: try json[.about]?.decode(to: String?.self),
140+
created: try json[.created].decode(),
141+
updated: try json[.updated].decode(),
142+
pushed: try json[.pushed].decode()))
143+
}
144+
}

‎Sources/SwiftinitServer/Plugins/GitHubClient.Connection (ext).swift

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ extension GitHubClient<GitHub.API>.Connection
3030
func crawl(
3131
owner:String,
3232
repo:String,
33-
pat:String) async throws -> GitHubPlugin.CrawlerResponse
33+
pat:String) async throws -> GitHubPlugin.RepoMonitorResponse
3434
{
3535
let query:JSON = .object
3636
{
@@ -78,4 +78,55 @@ extension GitHubClient<GitHub.API>.Connection
7878
}
7979
return try await self.post(query: "\(query)", with: pat)
8080
}
81+
82+
func search(repos search:String,
83+
limit:Int = 1000,
84+
pat:String) async throws -> GitHubPlugin.RepoTelescopeResponse
85+
{
86+
let query:JSON = .object
87+
{
88+
$0["query"] = """
89+
query
90+
{
91+
search(query: \(search), type: REPOSITORY, first: \(limit))
92+
{
93+
nodes
94+
{
95+
... on Repository
96+
{
97+
id: databaseId
98+
owner { login }
99+
name
100+
101+
license: licenseInfo { id: spdxId, name }
102+
topics: repositoryTopics(first: 16)
103+
{
104+
nodes { topic { name } }
105+
}
106+
master: defaultBranchRef { name }
107+
108+
watchers(first: 0) { count: totalCount }
109+
forks: forkCount
110+
stars: stargazerCount
111+
size: diskUsage
112+
113+
archived: isArchived
114+
disabled: isDisabled
115+
fork: isFork
116+
117+
homepage: homepageUrl
118+
about: description
119+
120+
created: createdAt
121+
updated: updatedAt
122+
pushed: pushedAt
123+
}
124+
}
125+
}
126+
}
127+
"""
128+
}
129+
130+
return try await self.post(query: "\(query)", with: pat)
131+
}
81132
}

‎Sources/SwiftinitServer/Plugins/GitHubCrawler.swift

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import GitHubAPI
2+
import GitHubClient
3+
import HTTPServer
4+
import MongoDB
5+
import UnidocDB
6+
import UnixTime
7+
8+
protocol GitHubCrawler
9+
{
10+
static
11+
var interval:Duration { get }
12+
13+
var api:GitHubClient<GitHub.API> { get }
14+
15+
func crawl(updating server:Swiftinit.ServerLoop,
16+
over connection:GitHubClient<GitHub.API>.Connection,
17+
with session:Mongo.Session) async throws
18+
}
19+
extension GitHubCrawler
20+
{
21+
func run(alongside server:Swiftinit.ServerLoop) async throws
22+
{
23+
while true
24+
{
25+
async
26+
let cooldown:Void = Task.sleep(for: Self.interval)
27+
28+
do
29+
{
30+
let session:Mongo.Session = try await .init(from: server.db.sessions)
31+
try await self.api.connect
32+
{
33+
try await self.crawl(updating: server, over: $0, with: session)
34+
}
35+
}
36+
catch let error as any GitHubRateLimitError
37+
{
38+
try await Task.sleep(for: error.until - .now())
39+
}
40+
catch let error
41+
{
42+
Log[.warning] = "GitHub crawling error: \(error)"
43+
server.atomics.errorsCrawling.wrappingIncrement(ordering: .relaxed)
44+
}
45+
46+
try await cooldown
47+
}
48+
}
49+
}

0 commit comments

Comments
 (0)