Merge pull request #49302 from JuliaLang/backports-release-1.9

Backports for 1.9.0-rc3

Author: KristofferC

NEWS.md

@@ -139,8 +139,8 @@ Standard library changes
 * The contextual module which is active in the REPL can be changed (it is `Main` by default),
   via the `REPL.activate(::Module)` function or via typing the module in the REPL and pressing
   the keybinding Alt-m ([#33872]).
-* An "IPython mode" which mimics the behaviour of the prompts and storing the evaluated result in `Out` can be
-  activated with `REPL.ipython_mode!()`. See the manual for how to enable this at startup ([#46474]).
+* A "numbered prompt" mode which prints numbers for each input and output and stores evaluated results in `Out` can be
+  activated with `REPL.numbered_prompt!()`. See the manual for how to enable this at startup ([#46474]).
 * Tab completion displays available keyword arguments ([#43536]).
 
 #### SuiteSparse

base/array.jl

@@ -267,7 +267,7 @@ end
 """
     unsafe_copyto!(dest::Array, do, src::Array, so, N)
 
-Copy `N` elements from a source array to a destination, starting at offset `so` in the
+Copy `N` elements from a source array to a destination, starting at the linear index `so` in the
 source and `do` in the destination (1-indexed).
 
 The `unsafe` prefix on this function indicates that no validation is performed to ensure
@@ -307,8 +307,8 @@ unsafe_copyto!(dest::Array, doffs, src::Array, soffs, n) =
 """
     copyto!(dest, do, src, so, N)
 
-Copy `N` elements from collection `src` starting at offset `so`, to array `dest` starting at
-offset `do`. Return `dest`.
+Copy `N` elements from collection `src` starting at the linear index `so`, to array `dest` starting at
+the index `do`. Return `dest`.
 """
 function copyto!(dest::Array, doffs::Integer, src::Array, soffs::Integer, n::Integer)
     return _copyto_impl!(dest, doffs, src, soffs, n)

base/binaryplatforms.jl

@@ -1066,12 +1066,20 @@ function select_platform(download_info::Dict, platform::AbstractPlatform = HostP
         return nothing
     end
 
-    # At this point, we may have multiple possibilities.  E.g. if, in the future,
-    # Julia can be built without a direct dependency on libgfortran, we may match
-    # multiple tarballs that vary only within their libgfortran ABI.  To narrow it
-    # down, we just sort by triplet, then pick the last one.  This has the effect
-    # of generally choosing the latest release (e.g. a `libgfortran5` tarball
-    # rather than a `libgfortran3` tarball)
+    # At this point, we may have multiple possibilities.  We now engage a multi-
+    # stage selection algorithm, where we first choose simpler matches over more
+    # complex matches.  We define a simpler match as one that has fewer tags
+    # overall.  As these candidate matches have already been filtered to match
+    # the given platform, the only other tags that exist are ones that are in
+    # addition to the tags declared by the platform. Hence, selecting the
+    # minimum in number of tags is equivalent to selecting the closest match.
+    min_tag_count = minimum(length(tags(p)) for p in ps)
+    filter!(p -> length(tags(p)) == min_tag_count, ps)
+
+    # Now we _still_ may continue to have multiple matches, so we now simply sort
+    # the candidate matches by their triplets and take the last one, so as to
+    # generally choose the latest release (e.g. a `libgfortran5` tarball over a
+    # `libgfortran3` tarball).
     p = last(sort(ps, by = p -> triplet(p)))
     return download_info[p]
 end

base/file.jl

@@ -105,7 +105,7 @@ if Sys.iswindows()
     end
 else
     function cd(f::Function, dir::AbstractString)
-        fd = ccall(:open, Int32, (Cstring, Int32), :., 0)
+        fd = ccall(:open, Int32, (Cstring, Int32, UInt32...), :., 0)
         systemerror(:open, fd == -1)
         try
             cd(dir)

base/loading.jl

@@ -1080,6 +1080,7 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
 end
 
 function run_package_callbacks(modkey::PkgId)
+    run_extension_callbacks(modkey)
     assert_havelock(require_lock)
     unlock(require_lock)
     try
@@ -1188,9 +1189,12 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, <:An
     end
 end
 
+loading_extension::Bool = false
 function run_extension_callbacks(extid::ExtensionId)
     assert_havelock(require_lock)
     succeeded = try
+        # Used by Distributed to now load extensions in the package callback
+        global loading_extension = true
         _require_prelocked(extid.id)
         @debug "Extension $(extid.id.name) of $(extid.parentid.name) loaded"
         true
@@ -1200,59 +1204,57 @@ function run_extension_callbacks(extid::ExtensionId)
         @error "Error during loading of extension $(extid.id.name) of $(extid.parentid.name), \
                 use `Base.retry_load_extensions()` to retry." exception=errs
         false
+    finally
+        global loading_extension = false
     end
     return succeeded
 end
 
-function run_extension_callbacks()
+function run_extension_callbacks(pkgid::PkgId)
     assert_havelock(require_lock)
-    loaded_triggers = collect(intersect(keys(Base.loaded_modules), keys(Base.EXT_DORMITORY)))
-    sort!(loaded_triggers; by=x->x.uuid)
-    for pkgid in loaded_triggers
-        # take ownership of extids that depend on this pkgid
-        extids = pop!(EXT_DORMITORY, pkgid, nothing)
-        extids === nothing && continue
-        for extid in extids
-            if extid.ntriggers > 0
-                # It is possible that pkgid was loaded in an environment
-                # below the one of the parent. This will cause a load failure when the
-                # pkg ext tries to load the triggers. Therefore, check this first
-                # before loading the pkg ext.
-                pkgenv = Base.identify_package_env(extid.id, pkgid.name)
-                ext_not_allowed_load = false
-                if pkgenv === nothing
+    # take ownership of extids that depend on this pkgid
+    extids = pop!(EXT_DORMITORY, pkgid, nothing)
+    extids === nothing && return
+    for extid in extids
+        if extid.ntriggers > 0
+            # It is possible that pkgid was loaded in an environment
+            # below the one of the parent. This will cause a load failure when the
+            # pkg ext tries to load the triggers. Therefore, check this first
+            # before loading the pkg ext.
+            pkgenv = identify_package_env(extid.id, pkgid.name)
+            ext_not_allowed_load = false
+            if pkgenv === nothing
+                ext_not_allowed_load = true
+            else
+                pkg, env = pkgenv
+                path = locate_package(pkg, env)
+                if path === nothing
                     ext_not_allowed_load = true
-                else
-                    pkg, env = pkgenv
-                    path = Base.locate_package(pkg, env)
-                    if path === nothing
-                        ext_not_allowed_load = true
-                    end
-                end
-                if ext_not_allowed_load
-                    @debug "Extension $(extid.id.name) of $(extid.parentid.name) will not be loaded \
-                            since $(pkgid.name) loaded in environment lower in load path"
-                    # indicate extid is expected to fail
-                    extid.ntriggers *= -1
-                else
-                    # indicate pkgid is loaded
-                    extid.ntriggers -= 1
                 end
             end
-            if extid.ntriggers < 0
-                # indicate pkgid is loaded
-                extid.ntriggers += 1
-                succeeded = false
+            if ext_not_allowed_load
+                @debug "Extension $(extid.id.name) of $(extid.parentid.name) will not be loaded \
+                        since $(pkgid.name) loaded in environment lower in load path"
+                # indicate extid is expected to fail
+                extid.ntriggers *= -1
             else
-                succeeded = true
-            end
-            if extid.ntriggers == 0
-                # actually load extid, now that all dependencies are met,
-                # and record the result
-                succeeded = succeeded && run_extension_callbacks(extid)
-                succeeded || push!(EXT_DORMITORY_FAILED, extid)
+                # indicate pkgid is loaded
+                extid.ntriggers -= 1
             end
         end
+        if extid.ntriggers < 0
+            # indicate pkgid is loaded
+            extid.ntriggers += 1
+            succeeded = false
+        else
+            succeeded = true
+        end
+        if extid.ntriggers == 0
+            # actually load extid, now that all dependencies are met,
+            # and record the result
+            succeeded = succeeded && run_extension_callbacks(extid)
+            succeeded || push!(EXT_DORMITORY_FAILED, extid)
+        end
     end
     return
 end
@@ -1276,7 +1278,7 @@ function retry_load_extensions()
     end
     prepend!(EXT_DORMITORY_FAILED, failed)
     end
-    nothing
+    return
 end
 
 """
@@ -1510,11 +1512,11 @@ end
 """
     include_dependency(path::AbstractString)
 
-In a module, declare that the file specified by `path` (relative or absolute) is a
-dependency for precompilation; that is, the module will need to be recompiled if this file
-changes.
+In a module, declare that the file, directory, or symbolic link specified by `path`
+(relative or absolute) is a dependency for precompilation; that is, the module will need
+to be recompiled if the modification time of `path` changes.
 
-This is only needed if your module depends on a file that is not used via [`include`](@ref). It has
+This is only needed if your module depends on a path that is not used via [`include`](@ref). It has
 no effect outside of compilation.
 """
 function include_dependency(path::AbstractString)
@@ -1631,10 +1633,6 @@ function _require_prelocked(uuidkey::PkgId, env=nothing)
     else
         newm = root_module(uuidkey)
     end
-    # Load extensions when not precompiling and not in a nested package load
-    if JLOptions().incremental == 0 && isempty(package_locks)
-        run_extension_callbacks()
-    end
     return newm
 end
 
@@ -2790,7 +2788,7 @@ end
             end
             for chi in includes
                 f, ftime_req = chi.filename, chi.mtime
-                if !isfile(f)
+                if !ispath(f)
                     _f = fixup_stdlib_path(f)
                     if isfile(_f) && startswith(_f, Sys.STDLIB)
                         # mtime is changed by extraction

base/strings/lazy.jl

@@ -67,7 +67,7 @@ macro lazy_str(text)
     parts = Any[]
     lastidx = idx = 1
     while (idx = findnext('$', text, idx)) !== nothing
-        lastidx < idx && push!(parts, text[lastidx:idx-1])
+        lastidx < idx && push!(parts, text[lastidx:prevind(text, idx)])
         idx += 1
         expr, idx = Meta.parseatom(text, idx; filename=string(__source__.file))
         push!(parts, esc(expr))

base/task.jl

@@ -253,7 +253,7 @@ istaskfailed(t::Task) = (load_state_acquire(t) === task_state_failed)
 Threads.threadid(t::Task) = Int(ccall(:jl_get_task_tid, Int16, (Any,), t)+1)
 function Threads.threadpool(t::Task)
     tpid = ccall(:jl_get_task_threadpoolid, Int8, (Any,), t)
-    return tpid == 0 ? :default : :interactive
+    return Threads._tpid_to_sym(tpid)
 end
 
 task_result(t::Task) = t.result
@@ -786,7 +786,7 @@ function enq_work(t::Task)
         if Threads.threadpoolsize(tp) == 1
             # There's only one thread in the task's assigned thread pool;
             # use its work queue.
-            tid = (tp === :default) ? 1 : Threads.threadpoolsize(:default)+1
+            tid = (tp === :interactive) ? 1 : Threads.threadpoolsize(:interactive)+1
             ccall(:jl_set_task_tid, Cint, (Any, Cint), t, tid-1)
             push!(workqueue_for(tid), t)
         else

base/threadingconstructs.jl

@@ -39,14 +39,22 @@ function _nthreads_in_pool(tpid::Int8)
     return Int(unsafe_load(p, tpid + 1))
 end
 
+function _tpid_to_sym(tpid::Int8)
+    return tpid == 0 ? :interactive : :default
+end
+
+function _sym_to_tpid(tp::Symbol)
+    return tp === :interactive ? Int8(0) : Int8(1)
+end
+
 """
     Threads.threadpool(tid = threadid()) -> Symbol
 
 Returns the specified thread's threadpool; either `:default` or `:interactive`.
 """
 function threadpool(tid = threadid())
     tpid = ccall(:jl_threadpoolid, Int8, (Int16,), tid-1)
-    return tpid == 0 ? :default : :interactive
+    return _tpid_to_sym(tpid)
 end
 
 """
@@ -67,24 +75,39 @@ See also: `BLAS.get_num_threads` and `BLAS.set_num_threads` in the
 [`Distributed`](@ref man-distributed) standard library.
 """
 function threadpoolsize(pool::Symbol = :default)
-    if pool === :default
-        tpid = Int8(0)
-    elseif pool === :interactive
-        tpid = Int8(1)
+    if pool === :default || pool === :interactive
+        tpid = _sym_to_tpid(pool)
     else
         error("invalid threadpool specified")
     end
     return _nthreads_in_pool(tpid)
 end
 
+"""
+    threadpooltids(pool::Symbol)
+
+Returns a vector of IDs of threads in the given pool.
+"""
+function threadpooltids(pool::Symbol)
+    ni = _nthreads_in_pool(Int8(0))
+    if pool === :interactive
+        return collect(1:ni)
+    elseif pool === :default
+        return collect(ni+1:ni+_nthreads_in_pool(Int8(1)))
+    else
+        error("invalid threadpool specified")
+    end
+end
+
 function threading_run(fun, static)
     ccall(:jl_enter_threaded_region, Cvoid, ())
     n = threadpoolsize()
+    tid_offset = threadpoolsize(:interactive)
     tasks = Vector{Task}(undef, n)
     for i = 1:n
         t = Task(() -> fun(i)) # pass in tid
         t.sticky = static
-        static && ccall(:jl_set_task_tid, Cint, (Any, Cint), t, i-1)
+        static && ccall(:jl_set_task_tid, Cint, (Any, Cint), t, tid_offset + i-1)
         tasks[i] = t
         schedule(t)
     end
@@ -287,6 +310,15 @@ macro threads(args...)
     return _threadsfor(ex.args[1], ex.args[2], sched)
 end
 
+function _spawn_set_thrpool(t::Task, tp::Symbol)
+    tpid = _sym_to_tpid(tp)
+    if _nthreads_in_pool(tpid) == 0
+        tpid = _sym_to_tpid(:default)
+    end
+    ccall(:jl_set_task_threadpoolid, Cint, (Any, Int8), t, tpid)
+    nothing
+end
+
 """
     Threads.@spawn [:default|:interactive] expr
 
@@ -315,7 +347,7 @@ the variable's value in the current task.
     A threadpool may be specified as of Julia 1.9.
 """
 macro spawn(args...)
-    tpid = Int8(0)
+    tp = :default
     na = length(args)
     if na == 2
         ttype, ex = args
@@ -325,9 +357,9 @@ macro spawn(args...)
             # TODO: allow unquoted symbols
             ttype = nothing
         end
-        if ttype === :interactive
-            tpid = Int8(1)
-        elseif ttype !== :default
+        if ttype === :interactive || ttype === :default
+            tp = ttype
+        else
             throw(ArgumentError("unsupported threadpool in @spawn: $ttype"))
         end
     elseif na == 1
@@ -344,11 +376,7 @@ macro spawn(args...)
         let $(letargs...)
             local task = Task($thunk)
             task.sticky = false
-            local tpid_actual = $tpid
-            if _nthreads_in_pool(tpid_actual) == 0
-                tpid_actual = Int8(0)
-            end
-            ccall(:jl_set_task_threadpoolid, Cint, (Any, Int8), task, tpid_actual)
+            _spawn_set_thrpool(task, $(QuoteNode(tp)))
             if $(Expr(:islocal, var))
                 put!($var, task)
             end