Use pigz to create archives (#107)

Author: omus

Manifest.toml

@@ -28,9 +28,9 @@ deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[JLLWrappers]]
-git-tree-sha1 = "7cec881362e5b4e367ff0279dd99a06526d51a55"
+git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0"
 uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
-version = "1.1.2"
+version = "1.2.0"
 
 [[JSON]]
 deps = ["Dates", "Mmap", "Parsers", "Unicode"]
@@ -47,7 +47,7 @@ deps = ["Libdl"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
-deps = ["Printf"]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [[Libdl]]
@@ -75,10 +75,10 @@ uuid = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c"
 version = "0.1.0"
 
 [[Parsers]]
-deps = ["Dates", "Test"]
-git-tree-sha1 = "8077624b3c450b15c087944363606a6ba12f925e"
+deps = ["Dates"]
+git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "1.0.10"
+version = "1.0.15"
 
 [[Pkg]]
 deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
@@ -137,12 +137,18 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[Zlib_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6"
+git-tree-sha1 = "32085929ad61a5ed99abea288ec7242be392bb8b"
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.11+18"
+version = "1.2.12+0"
 
 [[p7zip_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
 git-tree-sha1 = "ee65cfa19bea645698a0224bfa216f2b1c8b559f"
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 version = "16.2.0+3"
+
+[[pigz_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
+git-tree-sha1 = "8c379a72c82099ceb4be53f4f427690376279052"
+uuid = "1bc43ea1-30af-5bc8-a9d4-c018457e6e3e"
+version = "2.5.0+0"

Project.toml

@@ -20,6 +20,7 @@ TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+pigz_jll = "1bc43ea1-30af-5bc8-a9d4-c018457e6e3e"
 
 [compat]
 CodecZlib = "0.5, 0.6, 0.7"

src/ArchiveUtils.jl

@@ -1,4 +1,6 @@
-using Downloads, Tar, p7zip_jll, SimpleBufferStream, SHA
+using Base: SHA1
+using Downloads, Tar, p7zip_jll, pigz_jll, SimpleBufferStream, SHA
+using Pkg.Artifacts: artifact_exists, artifact_path, query_override
 
 export unpack, list_tarball_files, verify, download_verify
 
@@ -146,3 +148,32 @@ function download_verify(url, hash, path)
         verify(path, hash) || error("Verification failed")
     end
 end
+
+# Stripped down copy of `Pkg.Artifacts.archive_artifact` that supports a custom `package`
+# function
+function _archive_artifact(hash::SHA1, tarball_path::String;
+                           honor_overrides::Bool=false,
+                           package::Function=_package_fast)
+
+    if !artifact_exists(hash)
+        error("Unable to archive artifact $(bytes2hex(hash.bytes)): does not exist!")
+    end
+
+    # Package it up
+    package(artifact_path(hash), tarball_path)
+
+    # Calculate its sha256 and return that
+    return open(tarball_path, "r") do io
+        return bytes2hex(sha256(io))
+    end
+end
+
+# Copy of `Pkg.PlatformEngines.package` but using `pigz` instead of `7z`
+function _package_fast(src_dir::AbstractString, tarball_path::AbstractString)
+    rm(tarball_path, force=true)
+    pigz() do pigz_exe
+        open(pipeline(`$pigz_exe -9`, stdout=tarball_path), write=true) do io
+            Tar.create(src_dir, io)
+        end
+    end
+end

src/Prefix.jl

@@ -173,7 +173,7 @@ function package(prefix::Prefix,
         @info("Tree hash of contents of $(basename(out_path)): $(tree_hash)")
     end
 
-    tarball_hash = archive_artifact(tree_hash, out_path; honor_overrides=false)
+    tarball_hash = _archive_artifact(tree_hash, out_path; honor_overrides=false)
     if verbose
         @info("SHA256 of $(basename(out_path)): $(tarball_hash)")
     end