Artifacts: avoid reflection hacks to access ensure_artifact_installed (#37844)

Packages should never access Base.loaded_modules() to call functions
from it, as it can be brittle and create future incompatibilities, so
instead we require the user to explicitly declare a dependency on the
lazy-download machinery, if they requiring the ability to use it (for
lazy artifacts).

As a deprecation, if the user has `using Pkg`, that will be used
instead, with a depwarn.

Author: vtjnash

NEWS.md

@@ -150,6 +150,8 @@ Standard library changes
 * The `Pkg.Artifacts` module has been imported as a separate standard library.  It is still available as
   `Pkg.Artifacts`, however starting from Julia v1.6+, packages may import simply `Artifacts` without importing
   all of `Pkg` alongside ([#37320]).
+* To download artifacts lazily, `LazyArtifacts` now must be explicitly listed as a dependency, to avoid needing the
+  support machinery to be available when it is not commonly needed ([#37844]).
 * `@time` now reports if the time presented included any compilation time, which is shown as a percentage ([#37678]).
 * `@varinfo` can now report non-exported objects within modules, look recursively into submodules, and return a sorted
   results table ([#38042]).

base/sysimg.jl

@@ -68,6 +68,9 @@ let
 
         # 4-depth packages
         :Pkg,
+
+        # 5-depth packages
+        :LazyArtifacts,
     ]
     maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)
 

stdlib/Artifacts/src/Artifacts.jl

@@ -524,7 +524,7 @@ function jointail(dir, tail)
     end
 end
 
-function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform)
+function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, @nospecialize(lazyartifacts))
     if haskey(Base.module_keys, __module__)
         # Process overrides for this UUID, if we know what it is
         process_overrides(artifact_dict, Base.module_keys[__module__].uuid)
@@ -538,10 +538,16 @@ function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dic
         end
     end
 
-    # If not, we need to download it.  We look up the Pkg module through `Base.loaded_modules()`
-    # then invoke `ensure_artifact_installed()`:
-    Pkg = first(filter(p-> p[1].name == "Pkg", Base.loaded_modules))[2]
-    return jointail(Pkg.Artifacts.ensure_artifact_installed(string(name), artifacts_toml; platform), path_tail)
+    # If not, try determining what went wrong:
+    meta = artifact_meta(name, artifact_dict, artifacts_toml; platform)
+    if meta !== nothing && get(meta, "lazy", false)
+        if lazyartifacts isa Module && isdefined(lazyartifacts, :ensure_artifact_installed)
+            nameof(lazyartifacts) === :Pkg && Base.depwarn("using Pkg instead of using LazyArtifacts is deprecated", :var"@artifact_str", force=true)
+            return jointail(lazyartifacts.ensure_artifact_installed(string(name), artifacts_toml; platform), path_tail)
+        end
+        error("Artifact $(repr(name)) is a lazy artifact; package developers must call `using LazyArtifacts` in $(__module__) before using lazy artifacts.")
+    end
+    error("Artifact $(repr(name)) was not installed correctly. Try `using Pkg; Pkg.instantiate()` to re-install all missing resources.")
 end
 
 """
@@ -607,11 +613,15 @@ end
 """
     macro artifact_str(name)
 
-Macro that is used to automatically ensure an artifact is installed, and return its
-location on-disk.  Automatically looks the artifact up by name in the project's
-`(Julia)Artifacts.toml` file.  Throws an error on inability to install the requested
-artifact.  If run in the REPL, searches for the toml file starting in the current
-directory, see `find_artifacts_toml()` for more.
+Return the on-disk path to an artifact. Automatically looks the artifact up by
+name in the project's `(Julia)Artifacts.toml` file. Throws an error on if the
+requested artifact is not present. If run in the REPL, searches for the toml
+file starting in the current directory, see `find_artifacts_toml()` for more.
+
+If the artifact is marked "lazy" and the package has `using LazyArtifacts`
+defined, the artifact will be downloaded on-demand with `Pkg` the first time
+this macro tries to compute the path. The files will then be left installed
+locally for later.
 
 If `name` contains a forward or backward slash, all elements after the first slash will
 be taken to be path names indexing into the artifact, allowing for an easy one-liner to
@@ -649,14 +659,20 @@ macro artifact_str(name, platform=nothing)
     # Invalidate calling .ji file if Artifacts.toml file changes
     Base.include_dependency(artifacts_toml)
 
+    # Check if the user has provided `LazyArtifacts`, and thus supports lazy artifacts
+    lazyartifacts = isdefined(__module__, :LazyArtifacts) ? GlobalRef(__module__, :LazyArtifacts) : nothing
+    if lazyartifacts === nothing && isdefined(__module__, :Pkg)
+        lazyartifacts = GlobalRef(__module__, :Pkg) # deprecated
+    end
+
     # If `name` is a constant, (and we're using the default `Platform`) we can actually load
     # and parse the `Artifacts.toml` file now, saving the work from runtime.
     if isa(name, AbstractString) && platform === nothing
         # To support slash-indexing, we need to split the artifact name from the path tail:
         platform = HostPlatform()
-        local artifact_name, artifact_path_tail, hash = artifact_slash_lookup(name, artifact_dict, artifacts_toml, platform)
+        artifact_name, artifact_path_tail, hash = artifact_slash_lookup(name, artifact_dict, artifacts_toml, platform)
         return quote
-            Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), $(artifact_name), $(artifact_path_tail), $(artifact_dict), $(hash), $(platform))
+            Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), $(artifact_name), $(artifact_path_tail), $(artifact_dict), $(hash), $(platform), $(lazyartifacts))
         end
     else
         if platform === nothing
@@ -665,7 +681,7 @@ macro artifact_str(name, platform=nothing)
         return quote
             local platform = $(esc(platform))
             local artifact_name, artifact_path_tail, hash = artifact_slash_lookup($(esc(name)), $(artifact_dict), $(artifacts_toml), platform)
-            Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), artifact_name, artifact_path_tail, $(artifact_dict), hash, platform)
+            Base.invokelatest(_artifact_str, $(__module__), $(artifacts_toml), artifact_name, artifact_path_tail, $(artifact_dict), hash, platform, $(lazyartifacts))
         end
     end
 end

stdlib/Artifacts/test/.gitignore

@@ -0,0 +1 @@
+/artifacts/

stdlib/Artifacts/test/refresh_artifacts.jl

@@ -0,0 +1,23 @@
+using Artifacts: with_artifacts_directory
+# using Pkg.Artifacts: ensure_all_artifacts_installed
+using Pkg.Artifacts: load_artifacts_toml, ensure_artifact_installed
+let
+    tempdir = joinpath(@__DIR__, "artifacts")
+    toml = joinpath(@__DIR__, "Artifacts.toml")
+    unused = Base.BinaryPlatforms.Platform(string(Sys.ARCH), "linux")
+    with_artifacts_directory(tempdir) do
+        # ensure_all_artifacts_installed(toml; include_lazy=false)
+        dict = load_artifacts_toml(toml)
+        for (name, meta) in dict
+            if meta isa Array
+                for meta in meta
+                    get(meta, "lazy", false) && continue
+                    ensure_artifact_installed(name, meta, toml; platform=unused)
+                end
+            else; meta::Dict
+                get(meta, "lazy", false) && continue
+                ensure_artifact_installed(name, meta, toml; platform=unused)
+            end
+        end
+    end
+end

stdlib/Artifacts/test/runtests.jl

@@ -3,6 +3,9 @@
 using Artifacts, Test, Base.BinaryPlatforms
 using Artifacts: with_artifacts_directory, pack_platform!, unpack_platform
 
+# prepare for the package tests by ensuring the required artifacts are downloaded now
+run(addenv(`$(Base.julia_cmd()) --color=no $(joinpath(@__DIR__, "refresh_artifacts.jl"))`, "TERM"=>"dumb"))
+
 @testset "Artifact Paths" begin
     mktempdir() do tempdir
         with_artifacts_directory(tempdir) do
@@ -78,45 +81,43 @@ end
 end
 
 @testset "Artifact Slash-indexing" begin
-    mktempdir() do tempdir
-        with_artifacts_directory(tempdir) do
-            exeext = Sys.iswindows() ? ".exe" : ""
-
-            # simple lookup, gives us the directory for `c_simple` for the current architecture
-            c_simple_dir = artifact"c_simple"
-            @test isdir(c_simple_dir)
-            c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)")
-            @test isfile(c_simple_exe_path)
-
-            # Simple slash-indexed lookup
-            c_simple_bin_path = artifact"c_simple/bin"
-            @test isdir(c_simple_bin_path)
-            # Test that forward and backward slash are equivalent
-            @test artifact"c_simple\\bin" == artifact"c_simple/bin"
-
-            # Dynamically-computed lookup; not done at compile-time
-            generate_artifact_name() = "c_simple"
-            c_simple_dir = @artifact_str(generate_artifact_name())
-            @test isdir(c_simple_dir)
-            c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)")
-            @test isfile(c_simple_exe_path)
-
-            # Dynamically-computed slash-indexing:
-            generate_bin_path(pathsep) = "c_simple$(pathsep)bin$(pathsep)c_simple$(exeext)"
-            @test isfile(@artifact_str(generate_bin_path("/")))
-            @test isfile(@artifact_str(generate_bin_path("\\")))
-        end
+    tempdir = joinpath(@__DIR__, "artifacts")
+    with_artifacts_directory(tempdir) do
+        exeext = Sys.iswindows() ? ".exe" : ""
+
+        # simple lookup, gives us the directory for `c_simple` for the current architecture
+        c_simple_dir = artifact"c_simple"
+        @test isdir(c_simple_dir)
+        c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)")
+        @test isfile(c_simple_exe_path)
+
+        # Simple slash-indexed lookup
+        c_simple_bin_path = artifact"c_simple/bin"
+        @test isdir(c_simple_bin_path)
+        # Test that forward and backward slash are equivalent
+        @test artifact"c_simple\\bin" == artifact"c_simple/bin"
+
+        # Dynamically-computed lookup; not done at compile-time
+        generate_artifact_name() = "c_simple"
+        c_simple_dir = @artifact_str(generate_artifact_name())
+        @test isdir(c_simple_dir)
+        c_simple_exe_path = joinpath(c_simple_dir, "bin", "c_simple$(exeext)")
+        @test isfile(c_simple_exe_path)
+
+        # Dynamically-computed slash-indexing:
+        generate_bin_path(pathsep) = "c_simple$(pathsep)bin$(pathsep)c_simple$(exeext)"
+        @test isfile(@artifact_str(generate_bin_path("/")))
+        @test isfile(@artifact_str(generate_bin_path("\\")))
     end
 end
 
 @testset "@artifact_str Platform passing" begin
-    mktempdir() do tempdir
-        with_artifacts_directory(tempdir) do
-            win64 = Platform("x86_64", "windows")
-            mac64 = Platform("x86_64", "macos")
-            @test basename(@artifact_str("c_simple", win64)) == "444cecb70ff39e8961dd33e230e151775d959f37"
-            @test basename(@artifact_str("c_simple", mac64)) == "7ba74e239348ea6c060f994c083260be3abe3095"
-        end
+    tempdir = joinpath(@__DIR__, "artifacts")
+    with_artifacts_directory(tempdir) do
+        win64 = Platform("x86_64", "windows")
+        mac64 = Platform("x86_64", "macos")
+        @test basename(@artifact_str("c_simple", win64)) == "444cecb70ff39e8961dd33e230e151775d959f37"
+        @test basename(@artifact_str("c_simple", mac64)) == "7ba74e239348ea6c060f994c083260be3abe3095"
     end
 end
 
@@ -131,3 +132,14 @@ end
     @test artifacts["c_simple"]["git-tree-sha1"] == "0c509b3302db90a9393d6036c3ffcd14d190523d"
     @test artifacts["socrates"]["git-tree-sha1"] == "43563e7631a7eafae1f9f8d9d332e3de44ad7239"
 end
+
+@testset "@artifact_str install errors" begin
+    mktempdir() do tempdir
+        with_artifacts_directory(tempdir) do
+            ex = @test_throws ErrorException artifact"c_simple"
+            @test startswith(ex.value.msg, "Artifact \"c_simple\" was not installed correctly. ")
+            ex = @test_throws ErrorException artifact"socrates"
+            @test startswith(ex.value.msg, "Artifact \"socrates\" is a lazy artifact; ")
+        end
+    end
+end

stdlib/LazyArtifacts/Project.toml

@@ -0,0 +1,12 @@
+name = "LazyArtifacts"
+uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+
+[deps]
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

stdlib/LazyArtifacts/src/LazyArtifacts.jl

@@ -0,0 +1,15 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+module LazyArtifacts
+
+# reexport the Artifacts API
+using Artifacts: Artifacts,
+       artifact_exists, artifact_path, artifact_meta, artifact_hash,
+       select_downloadable_artifacts, find_artifacts_toml, @artifact_str
+export artifact_exists, artifact_path, artifact_meta, artifact_hash,
+       select_downloadable_artifacts, find_artifacts_toml, @artifact_str
+
+# define a function for satisfying lazy Artifact downloads
+using Pkg.Artifacts: ensure_artifact_installed
+
+end

stdlib/LazyArtifacts/test/Artifacts.toml

@@ -0,0 +1 @@
+../../Artifacts/test/Artifacts.toml

stdlib/LazyArtifacts/test/runtests.jl

@@ -0,0 +1,25 @@
+using LazyArtifacts
+using Test
+
+mktempdir() do tempdir
+    LazyArtifacts.Artifacts.with_artifacts_directory(tempdir) do
+        socrates_dir = artifact"socrates"
+        @test isdir(socrates_dir)
+        ex = @test_throws ErrorException artifact"c_simple"
+        @test startswith(ex.value.msg, "Artifact \"c_simple\" was not installed correctly. ")
+    end
+end
+
+# Need to set depwarn flag before testing deprecations
+@test success(run(setenv(`$(Base.julia_cmd()) --depwarn=no --startup-file=no -e '
+    using Artifacts, Pkg
+    using Test
+    mktempdir() do tempdir
+        Artifacts.with_artifacts_directory(tempdir) do
+            socrates_dir = @test_logs(
+                    (:warn, "using Pkg instead of using LazyArtifacts is deprecated"),
+                    artifact"socrates")
+            @test isdir(socrates_dir)
+        end
+    end'`,
+    dir=@__DIR__)))