Skip to content

Commit 0e094a8

Browse files
authored
prevent doing excessive file system operations in require calls (JuliaLang#40890)
1 parent bfa4d38 commit 0e094a8

File tree

2 files changed

+118
-52
lines changed

2 files changed

+118
-52
lines changed

base/initdefs.jl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,8 @@ Return the fully expanded value of [`LOAD_PATH`](@ref) that is searched for proj
313313
packages.
314314
"""
315315
function load_path()
316+
cache = LOADING_CACHE[]
317+
cache !== nothing && return cache.load_path
316318
paths = String[]
317319
for env in LOAD_PATH
318320
path = load_path_expand(env)

base/loading.jl

Lines changed: 116 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -129,12 +129,21 @@ end
129129
const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab")
130130

131131
function dummy_uuid(project_file::String)
132+
cache = LOADING_CACHE[]
133+
if cache !== nothing
134+
uuid = get(cache.dummy_uuid, project_file, nothing)
135+
uuid === nothing || return uuid
136+
end
132137
project_path = try
133138
realpath(project_file)
134139
catch
135140
project_file
136141
end
137-
return uuid5(ns_dummy_uuid, project_path)
142+
uuid = uuid5(ns_dummy_uuid, project_path)
143+
if cache !== nothing
144+
cache.dummy_uuid[project_file] = uuid
145+
end
146+
return uuid
138147
end
139148

140149
## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ##
@@ -210,6 +219,17 @@ function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict)
210219
return f.d
211220
end
212221

222+
struct LoadingCache
223+
load_path::Vector{String}
224+
dummy_uuid::Dict{String, UUID}
225+
env_project_file::Dict{String, Union{Bool, String}}
226+
project_file_manifest_path::Dict{String, Union{Nothing, String}}
227+
require_parsed::Set{String}
228+
end
229+
const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing)
230+
LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set())
231+
232+
213233
struct TOMLCache
214234
p::TOML.Parser
215235
d::Dict{String, CachedTOMLDict}
@@ -220,14 +240,25 @@ const TOML_LOCK = ReentrantLock()
220240
parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, TOML_LOCK)
221241
function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock)
222242
lock(toml_lock) do
223-
if !haskey(toml_cache.d, project_file)
243+
cache = LOADING_CACHE[]
244+
dd = if !haskey(toml_cache.d, project_file)
224245
d = CachedTOMLDict(toml_cache.p, project_file)
225246
toml_cache.d[project_file] = d
226-
return d.d
247+
d.d
227248
else
228249
d = toml_cache.d[project_file]
229-
return get_updated_dict(toml_cache.p, d)
250+
# We are in a require call and have already parsed this TOML file
251+
# assume that it is unchanged to avoid hitting disk
252+
if cache !== nothing && project_file in cache.require_parsed
253+
d.d
254+
else
255+
get_updated_dict(toml_cache.p, d)
256+
end
257+
end
258+
if cache !== nothing
259+
push!(cache.require_parsed, project_file)
230260
end
261+
return dd
231262
end
232263
end
233264

@@ -352,16 +383,29 @@ const preferences_names = ("JuliaLocalPreferences.toml", "LocalPreferences.toml"
352383
# - `true`: `env` is an implicit environment
353384
# - `path`: the path of an explicit project file
354385
function env_project_file(env::String)::Union{Bool,String}
386+
cache = LOADING_CACHE[]
387+
if cache !== nothing
388+
project_file = get(cache.env_project_file, env, nothing)
389+
project_file === nothing || return project_file
390+
end
355391
if isdir(env)
356392
for proj in project_names
357-
project_file = joinpath(env, proj)
358-
isfile_casesensitive(project_file) && return project_file
393+
maybe_project_file = joinpath(env, proj)
394+
if isfile_casesensitive(maybe_project_file)
395+
project_file = maybe_project_file
396+
break
397+
end
359398
end
360-
return true
399+
project_file =true
361400
elseif basename(env) in project_names && isfile_casesensitive(env)
362-
return env
401+
project_file = env
402+
else
403+
project_file = false
363404
end
364-
return false
405+
if cache !== nothing
406+
cache.env_project_file[env] = project_file
407+
end
408+
return project_file
365409
end
366410

367411
function project_deps_get(env::String, name::String)::Union{Nothing,PkgId}
@@ -415,10 +459,9 @@ end
415459

416460
# find project file's top-level UUID entry (or nothing)
417461
function project_file_name_uuid(project_file::String, name::String)::PkgId
418-
uuid = dummy_uuid(project_file)
419462
d = parsed_toml(project_file)
420463
uuid′ = get(d, "uuid", nothing)::Union{String, Nothing}
421-
uuid′ === nothing || (uuid = UUID(uuid′))
464+
uuid = uuid=== nothing ? dummy_uuid(project_file) : UUID(uuid′)
422465
name = get(d, "name", name)::String
423466
return PkgId(uuid, name)
424467
end
@@ -430,18 +473,34 @@ end
430473

431474
# find project file's corresponding manifest file
432475
function project_file_manifest_path(project_file::String)::Union{Nothing,String}
476+
cache = LOADING_CACHE[]
477+
if cache !== nothing
478+
manifest_path = get(cache.project_file_manifest_path, project_file, missing)
479+
manifest_path === missing || return manifest_path
480+
end
433481
dir = abspath(dirname(project_file))
434482
d = parsed_toml(project_file)
435483
explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
484+
manifest_path = nothing
436485
if explicit_manifest !== nothing
437486
manifest_file = normpath(joinpath(dir, explicit_manifest))
438-
isfile_casesensitive(manifest_file) && return manifest_file
487+
if isfile_casesensitive(manifest_file)
488+
manifest_path = manifest_file
489+
end
439490
end
440-
for mfst in manifest_names
441-
manifest_file = joinpath(dir, mfst)
442-
isfile_casesensitive(manifest_file) && return manifest_file
491+
if manifest_path === nothing
492+
for mfst in manifest_names
493+
manifest_file = joinpath(dir, mfst)
494+
if isfile_casesensitive(manifest_file)
495+
manifest_path = manifest_file
496+
break
497+
end
498+
end
443499
end
444-
return nothing
500+
if cache !== nothing
501+
cache.project_file_manifest_path[project_file] = manifest_path
502+
end
503+
return manifest_path
445504
end
446505

447506
# given a directory (implicit env from LOAD_PATH) and a name,
@@ -576,10 +635,10 @@ function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry::
576635
hash = SHA1(hash)
577636
# Keep the 4 since it used to be the default
578637
uuid = pkg.uuid::UUID # checked within `explicit_manifest_uuid_path`
579-
for slug in (version_slug(uuid, hash, 4), version_slug(uuid, hash))
638+
for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4))
580639
for depot in DEPOT_PATH
581-
path = abspath(depot, "packages", pkg.name, slug)
582-
ispath(path) && return path
640+
path = joinpath(depot, "packages", pkg.name, slug)
641+
ispath(path) && return abspath(path)
583642
end
584643
end
585644
return nothing
@@ -876,42 +935,47 @@ For more details regarding code loading, see the manual sections on [modules](@r
876935
[parallel computing](@ref code-availability).
877936
"""
878937
function require(into::Module, mod::Symbol)
879-
uuidkey = identify_package(into, String(mod))
880-
# Core.println("require($(PkgId(into)), $mod) -> $uuidkey")
881-
if uuidkey === nothing
882-
where = PkgId(into)
883-
if where.uuid === nothing
884-
throw(ArgumentError("""
885-
Package $mod not found in current path:
886-
- Run `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.
887-
"""))
888-
else
889-
s = """
890-
Package $(where.name) does not have $mod in its dependencies:
891-
- If you have $(where.name) checked out for development and have
892-
added $mod as a dependency but haven't updated your primary
893-
environment's manifest file, try `Pkg.resolve()`.
894-
- Otherwise you may need to report an issue with $(where.name)"""
895-
896-
uuidkey = identify_package(PkgId(string(into)), String(mod))
897-
uuidkey === nothing && throw(ArgumentError(s))
898-
899-
# fall back to toplevel loading with a warning
900-
if !(where in modules_warned_for)
901-
@warn string(
902-
full_warning_showed[] ? "" : s, "\n",
903-
string("Loading $(mod) into $(where.name) from project dependency, ",
904-
"future warnings for $(where.name) are suppressed.")
905-
) _module = nothing _file = nothing _group = nothing
906-
push!(modules_warned_for, where)
938+
LOADING_CACHE[] = LoadingCache()
939+
try
940+
uuidkey = identify_package(into, String(mod))
941+
# Core.println("require($(PkgId(into)), $mod) -> $uuidkey")
942+
if uuidkey === nothing
943+
where = PkgId(into)
944+
if where.uuid === nothing
945+
throw(ArgumentError("""
946+
Package $mod not found in current path:
947+
- Run `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.
948+
"""))
949+
else
950+
s = """
951+
Package $(where.name) does not have $mod in its dependencies:
952+
- If you have $(where.name) checked out for development and have
953+
added $mod as a dependency but haven't updated your primary
954+
environment's manifest file, try `Pkg.resolve()`.
955+
- Otherwise you may need to report an issue with $(where.name)"""
956+
957+
uuidkey = identify_package(PkgId(string(into)), String(mod))
958+
uuidkey === nothing && throw(ArgumentError(s))
959+
960+
# fall back to toplevel loading with a warning
961+
if !(where in modules_warned_for)
962+
@warn string(
963+
full_warning_showed[] ? "" : s, "\n",
964+
string("Loading $(mod) into $(where.name) from project dependency, ",
965+
"future warnings for $(where.name) are suppressed.")
966+
) _module = nothing _file = nothing _group = nothing
967+
push!(modules_warned_for, where)
968+
end
969+
full_warning_showed[] = true
907970
end
908-
full_warning_showed[] = true
909971
end
972+
if _track_dependencies[]
973+
push!(_require_dependencies, (into, binpack(uuidkey), 0.0))
974+
end
975+
return require(uuidkey)
976+
finally
977+
LOADING_CACHE[] = nothing
910978
end
911-
if _track_dependencies[]
912-
push!(_require_dependencies, (into, binpack(uuidkey), 0.0))
913-
end
914-
return require(uuidkey)
915979
end
916980

917981
mutable struct PkgOrigin

0 commit comments

Comments
 (0)