support deflate symbolgraph compression

Author: tayloraswift

Package.resolved

@@ -100,6 +100,8 @@ let package:Package = .init(
             from: "0.5.0")),
         .package(url: "https://github.com/tayloraswift/swift-mongodb", .upToNextMinor(
             from: "0.10.5")),
+        .package(url: "https://github.com/tayloraswift/swift-png", .upToNextMinor(
+            from: "4.1.0")),
 
         .package(url: "https://github.com/apple/swift-atomics", .upToNextMinor(
             from: "1.2.0")),
@@ -595,6 +597,7 @@ let package:Package = .init(
                 .target(name: "SwiftinitAssets"),
                 .target(name: "SwiftinitPages"),
                 .target(name: "SwiftinitPlugins"),
+                .product(name: "LZ77", package: "swift-png"),
             ]),
 
 

Package.swift

@@ -1,5 +1,6 @@
 import BSON
 import HTTP
+import LZ77
 import MongoDB
 import S3
 import S3Client
@@ -29,6 +30,20 @@ extension Swiftinit.GraphStorageEndpoint:BlockingEndpoint
             if  let bucket:AWS.S3.Bucket = server.bucket,
                 let bson:[UInt8] = snapshot.move()
             {
+                var deflator:LZ77.Deflator = .init(format: .zlib,
+                    level: 7,
+                    hint: 128 << 10)
+
+                deflator.push((consume bson)[...], last: true)
+
+                var bson:[UInt8] = []
+                while let part:[UInt8] = deflator.pull()
+                {
+                    bson += part
+                }
+
+                snapshot.type = .bson_zz
+
                 let s3:AWS.S3.Client = .init(threads: server.context.threads,
                     niossl: server.context.niossl,
                     bucket: bucket)

Sources/SwiftinitServer/Endpoints/Procedural/Swiftinit.GraphStorageEndpoint.swift

@@ -1,4 +1,5 @@
 import BSON
+import LZ77
 import S3
 import S3Client
 import SymbolGraphs
@@ -21,6 +22,19 @@ extension Swiftinit.GraphLoader:Unidoc.GraphLoader
 {
     func load(graph:Unidoc.GraphPath) async throws -> [UInt8]
     {
-        try await self.s3.get(path: "\(graph)")
+        var bytes:[UInt8] = try await self.s3.get(path: "\(graph)")
+
+        switch graph.type
+        {
+        case .bson:
+            break
+
+        case .bson_zz:
+            var inflator:LZ77.Inflator = .init(format: .zlib)
+            try inflator.push((consume bytes)[...])
+            bytes = inflator.pull()
+        }
+
+        return bytes
     }
 }

Sources/SwiftinitServer/Plugins/Swiftinit.GraphLoader.swift

@@ -9,6 +9,8 @@ extension Unidoc
     {
         /// Uncompressed BSON.
         case bson = 0
+        /// Zlib-compressed BSON.
+        case bson_zz = 1
     }
 }
 extension Unidoc.GraphType:CustomStringConvertible
@@ -18,7 +20,8 @@ extension Unidoc.GraphType:CustomStringConvertible
     {
         switch self
         {
-        case .bson: "bson"
+        case .bson:     "bson"
+        case .bson_zz:  "bson.zz"
         }
     }
 }

Sources/UnidocRecords/Packages/Unidoc.GraphType.swift

@@ -107,7 +107,6 @@ extension Unidoc.Snapshot
         let document:BSON.Document = .init(encoding: inline)
 
         self.size = Int64.init(document.bytes.count)
-        self.type = .bson
         self.inline = nil
 
         return document.bytes