From 794764bf6c9a2c15f361199257864e2db647c3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 18 Apr 2025 12:39:14 -0400 Subject: [PATCH 01/17] Revise external method tables --- src/lowered.jl | 6 +-- src/packagedef.jl | 33 +++++++------- test/runtests.jl | 107 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 113 insertions(+), 33 deletions(-) diff --git a/src/lowered.jl b/src/lowered.jl index 083ec6a2..5e308788 100644 --- a/src/lowered.jl +++ b/src/lowered.jl @@ -16,7 +16,7 @@ end # This defines the API needed to store signatures using methods_by_execution! # This default version is simple and only used for testing purposes. # The "real" one is CodeTrackingMethodInfo in Revise.jl. -const MethodInfo = IdDict{Type,LineNumberNode} +const MethodInfo = IdDict{Pair{<:Union{Nothing, MethodTable},<:Type},LineNumberNode} add_signature!(methodinfo::MethodInfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln) push_expr!(methodinfo::MethodInfo, mod::Module, ex::Expr) = methodinfo pop_expr!(methodinfo::MethodInfo) = methodinfo @@ -464,13 +464,13 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra uT = Base.unwrap_unionall(T)::DataType ft = uT.types sig1 = Tuple{Base.rewrap_unionall(Type{uT}, T), Any[Any for i in 1:length(ft)]...} - push!(signatures, sig1) + push!(signatures, nothing => sig1) sig2 = Base.rewrap_unionall(Tuple{Type{T}, ft...}, T) while T isa UnionAll sig2 isa UnionAll || (sig2 = sig1; break) # sig2 doesn't define all parameters, so drop it T = T.body end - sig1 == sig2 || push!(signatures, sig2) + sig1 == sig2 || push!(signatures, nothing => sig2) for sig in signatures add_signature!(methodinfo, sig, lnn) end diff --git a/src/packagedef.jl b/src/packagedef.jl index 813cf982..f404c891 100644 --- a/src/packagedef.jl +++ b/src/packagedef.jl @@ -4,7 +4,7 @@ using FileWatching, REPL, UUIDs import LibGit2 using Base: PkgId using Base.Meta: isexpr -using Core: CodeInfo +using Core: CodeInfo, MethodTable export revise, includet, entr, MethodSummary @@ -286,14 +286,14 @@ function delete_missing!(exs_sigs_old::ExprsSigs, exs_sigs_new) haskey(exs_sigs_new, ex) && continue # ex was deleted sigs === nothing && continue - for sig in sigs - ret = Base._methods_by_ftype(sig, -1, Base.get_world_counter()) + for (mt, sig) in sigs + ret = Base._methods_by_ftype(sig, mt, -1, Base.get_world_counter()) success = false if !isempty(ret) m = get_method_from_match(ret[end]) # the last method returned is the least-specific that matches, and thus most likely to be type-equal methsig = m.sig if sig <: methsig && methsig <: sig - locdefs = get(CodeTracking.method_info, sig, nothing) + locdefs = get(CodeTracking.method_info, mt => sig, nothing) if isa(locdefs, Vector{Tuple{LineNumberNode,Expr}}) if length(locdefs) > 1 # Just delete this reference but keep the method @@ -312,14 +312,14 @@ function delete_missing!(exs_sigs_old::ExprsSigs, exs_sigs_new) for get_workers in workers_functions for p in @invokelatest get_workers() try # guard against serialization errors if the type isn't defined on the worker - @invokelatest remotecall_impl(Core.eval, p, Main, :(delete_method_by_sig($sig))) + @invokelatest remotecall_impl(Core.eval, p, Main, :(delete_method_by_sig($mt, $sig))) catch end end end Base.delete_method(m) # Remove the entries from CodeTracking data - delete!(CodeTracking.method_info, sig) + delete!(CodeTracking.method_info, mt => sig) # Remove frame from JuliaInterpreter, if applicable. Otherwise debuggers # may erroneously work with outdated code (265-like problems) if haskey(JuliaInterpreter.framedict, m) @@ -1161,7 +1161,7 @@ function get_def(method::Method; modified_files=revision_queue) # We need to find the right file. if method.module == Base || method.module == Core || method.module == Core.Compiler @warn "skipping $method to avoid parsing too much code" - CodeTracking.invoked_setindex!(CodeTracking.method_info, method.sig, missing) + CodeTracking.invoked_setindex!(CodeTracking.method_info, CodeTracking.method_info_key(method), missing) return false end parentfile, included_files = modulefiles(method.module) @@ -1178,15 +1178,15 @@ function get_def(method::Method; modified_files=revision_queue) def = get_def(method, pkgdata, file) def !== nothing && return true end - @warn "$(method.sig) was not found" + @warn "$(method.sig)$(isdefined(method, :external_mt) ? " (overlayed)" : "") was not found" # So that we don't call it again, store missingness info in CodeTracking - CodeTracking.invoked_setindex!(CodeTracking.method_info, method.sig, missing) + CodeTracking.invoked_setindex!(CodeTracking.method_info, CodeTracking.method_info_key(method), missing) return false end function get_def(method, pkgdata, filename) maybe_extract_sigs!(maybe_parse_from_cache!(pkgdata, filename)) - return get(CodeTracking.method_info, method.sig, nothing) + return get(CodeTracking.method_info, CodeTracking.method_info_key(method), nothing) end function get_tracked_id(id::PkgId; modified_files=revision_queue) @@ -1245,10 +1245,9 @@ function update_stacktrace_lineno!(trace) end if linfo isa Core.MethodInstance m = linfo.def - sigt = m.sig # Why not just call `whereis`? Because that forces tracking. This is being # clever by recognizing that these entries exist only if there have been updates. - updated = get(CodeTracking.method_info, sigt, nothing) + updated = get(CodeTracking.method_info, CodeTracking.method_info_key(m), nothing) if updated !== nothing lnn = updated[1][1] # choose the first entry by default lineoffset = lnn.line - m.line @@ -1263,7 +1262,7 @@ end function method_location(method::Method) # Why not just call `whereis`? Because that forces tracking. This is being # clever by recognizing that these entries exist only if there have been updates. - updated = get(CodeTracking.method_info, method.sig, nothing) + updated = get(CodeTracking.method_info, CodeTracking.method_info_key(method), nothing) if updated !== nothing lnn = updated[1][1] return lnn.file, lnn.line @@ -1321,16 +1320,16 @@ Revise itself does not need to be running on `p`. """ function init_worker(p::AbstractWorker) @invokelatest remotecall_impl(Core.eval, p, Main, quote - function whichtt(@nospecialize sig) - ret = Base._methods_by_ftype(sig, -1, Base.get_world_counter()) + function whichtt(mt::Union{Nothing, MethodTable}, @nospecialize sig) + ret = Base._methods_by_ftype(sig, mt, -1, Base.get_world_counter()) isempty(ret) && return nothing m = ret[end][3]::Method # the last method returned is the least-specific that matches, and thus most likely to be type-equal methsig = m.sig (sig <: methsig && methsig <: sig) || return nothing return m end - function delete_method_by_sig(@nospecialize sig) - m = whichtt(sig) + function delete_method_by_sig(mt::Union{Nothing, MethodTable}, @nospecialize sig) + m = whichtt(mt, sig) isa(m, Method) && Base.delete_method(m) end end) diff --git a/test/runtests.jl b/test/runtests.jl index a8c6c1b5..528561ae 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -266,7 +266,7 @@ end @test length(dvs) == 3 (def, val) = dvs[1] @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(square(x) = x^2))) - @test val == [Tuple{typeof(ReviseTest.square),Any}] + @test val == [nothing => Tuple{typeof(ReviseTest.square),Any}] @test Revise.firstline(Revise.unwrap(def)).line == 5 m = @which ReviseTest.square(1) @test m.line == 5 @@ -274,14 +274,14 @@ end @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) (def, val) = dvs[2] @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(cube(x) = x^3))) - @test val == [Tuple{typeof(ReviseTest.cube),Any}] + @test val == [nothing => Tuple{typeof(ReviseTest.cube),Any}] m = @which ReviseTest.cube(1) @test m.line == 7 @test whereis(m) == (tmpfile, 7) @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) (def, val) = dvs[3] @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(fourth(x) = x^4))) - @test val == [Tuple{typeof(ReviseTest.fourth),Any}] + @test val == [nothing => Tuple{typeof(ReviseTest.fourth),Any}] m = @which ReviseTest.fourth(1) @test m.line == 9 @test whereis(m) == (tmpfile, 9) @@ -291,7 +291,7 @@ end @test length(dvs) == 5 (def, val) = dvs[1] @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(mult2(x) = 2*x))) - @test val == [Tuple{typeof(ReviseTest.Internal.mult2),Any}] + @test val == [nothing => Tuple{typeof(ReviseTest.Internal.mult2),Any}] @test Revise.firstline(Revise.unwrap(def)).line == 13 m = @which ReviseTest.Internal.mult2(1) @test m.line == 11 @@ -299,7 +299,7 @@ end @test Revise.RelocatableExpr(definition(m)) == Revise.unwrap(def) (def, val) = dvs[2] @test isequal(Revise.unwrap(def), Revise.RelocatableExpr(:(mult3(x) = 3*x))) - @test val == [Tuple{typeof(ReviseTest.Internal.mult3),Any}] + @test val == [nothing => Tuple{typeof(ReviseTest.Internal.mult3),Any}] m = @which ReviseTest.Internal.mult3(1) @test m.line == 14 @test whereis(m) == (tmpfile, 14) @@ -327,10 +327,10 @@ end cmpdiff(logs[4], "Eval"; deltainfo=(ReviseTest, :(cube(x) = x^3))) cmpdiff(logs[5], "Eval"; deltainfo=(ReviseTest, :(fourth(x) = x^4))) stmpfile = Symbol(tmpfile) - cmpdiff(logs[6], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.mult2),Any}], LineNumberNode(11,stmpfile)=>LineNumberNode(13,stmpfile))) + cmpdiff(logs[6], "LineOffset"; deltainfo=(Any[nothing => Tuple{typeof(ReviseTest.Internal.mult2),Any}], LineNumberNode(11,stmpfile)=>LineNumberNode(13,stmpfile))) cmpdiff(logs[7], "Eval"; deltainfo=(ReviseTest.Internal, :(mult3(x) = 3*x))) - cmpdiff(logs[8], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.unchanged),Any}], LineNumberNode(18,stmpfile)=>LineNumberNode(19,stmpfile))) - cmpdiff(logs[9], "LineOffset"; deltainfo=(Any[Tuple{typeof(ReviseTest.Internal.unchanged2),Any}], LineNumberNode(20,stmpfile)=>LineNumberNode(21,stmpfile))) + cmpdiff(logs[8], "LineOffset"; deltainfo=(Any[nothing => Tuple{typeof(ReviseTest.Internal.unchanged),Any}], LineNumberNode(18,stmpfile)=>LineNumberNode(19,stmpfile))) + cmpdiff(logs[9], "LineOffset"; deltainfo=(Any[nothing => Tuple{typeof(ReviseTest.Internal.unchanged2),Any}], LineNumberNode(20,stmpfile)=>LineNumberNode(21,stmpfile))) @test length(Revise.actions(rlogger)) == 6 # by default LineOffset is skipped @test length(Revise.actions(rlogger; line=true)) == 9 @test_broken length(Revise.diffs(rlogger)) == 2 @@ -503,8 +503,8 @@ end m3 = first(methods(eval(fn3))) m3file = joinpath(dn, "subdir", "file3.jl") @test whereis(m3) == (m3file, 1) - @test signatures_at(m3file, 1) == [m3.sig] - @test signatures_at(eval(Symbol(modname)), joinpath("src", "subdir", "file3.jl"), 1) == [m3.sig] + @test signatures_at(m3file, 1) == [nothing => m3.sig] + @test signatures_at(eval(Symbol(modname)), joinpath("src", "subdir", "file3.jl"), 1) == [nothing => m3.sig] id = Base.PkgId(eval(Symbol(modname))) # for testing #596 pkgdata = Revise.pkgdatas[id] @@ -1398,7 +1398,7 @@ end """) @yry() @test MacroSigs.blah() == 1 - @test haskey(CodeTracking.method_info, (@which MacroSigs.blah()).sig) + @test haskey(CodeTracking.method_info, CodeTracking.method_info_key(@which MacroSigs.blah())) rm_precompile("MacroSigs") # Issue #568 (a macro *execution* bug) @@ -1896,8 +1896,8 @@ end ex2 = :(methspecificity(x::Integer) = 2) Core.eval(ReviseTestPrivate, ex1) Core.eval(ReviseTestPrivate, ex2) - exsig1 = Revise.RelocatableExpr(ex1)=>[Tuple{typeof(ReviseTestPrivate.methspecificity),Int}] - exsig2 = Revise.RelocatableExpr(ex2)=>[Tuple{typeof(ReviseTestPrivate.methspecificity),Integer}] + exsig1 = Revise.RelocatableExpr(ex1) => [nothing => Tuple{typeof(ReviseTestPrivate.methspecificity),Int}] + exsig2 = Revise.RelocatableExpr(ex2) => [nothing => Tuple{typeof(ReviseTestPrivate.methspecificity),Integer}] f_old, f_new = Revise.ExprsSigs(exsig1, exsig2), Revise.ExprsSigs(exsig2) Revise.delete_missing!(f_old, f_new) m = @which ReviseTestPrivate.methspecificity(1) @@ -3085,6 +3085,87 @@ end @test B670.y == 7 rm_precompile("B670") end + + do_test("External method tables") && @testset "External method tables" begin + function retval(m) + src = Base.uncompressed_ast(m) + node = src.code[1]::Core.ReturnNode + node.val + end + + testdir = newtestdir() + + unique_name(base) = Symbol(replace(lstrip(String(gensym(base)), '#'), '#' => '_')) + first_revision(name) = """ + module $name + + Base.Experimental.@MethodTable(method_table) + + foo() = 1 + Base.Experimental.@overlay method_table foo() = 2 + + bar() = foo() + + end + """ + second_revision(name) = """ + module $name + + Base.Experimental.@MethodTable(method_table) + + foo() = 1 + Base.Experimental.@overlay method_table foo() = 3 + + bar() = foo() + 1 + + end + """ + + name = unique_name(:ExternalMT) + dn = joinpath(testdir, "$name", "src") + mkpath(dn) + write(joinpath(dn, "$name.jl"), first_revision(name)) + sleep(mtimedelay) + @eval import $name + ExternalMT = @eval $name + foo, bar, mt = ExternalMT.foo, ExternalMT.bar, ExternalMT.method_table + sleep(mtimedelay) + @test foo() == 1 + @test bar() == 1 + (; ms) = Base.MethodList(mt) + @test retval(first(ms)) == 2 + write(joinpath(dn, "$name.jl"), second_revision(name)) + sleep(mtimedelay) + @yry() + @test foo() == 1 + @test bar() == 2 + (; ms) = Base.MethodList(mt) + @test retval(first(ms)) == 3 + + file = tempname() * ".jl" + name = unique_name(:ExternalMT_includet) + write(file, first_revision(name)) + sleep(mtimedelay) + Revise.track(@__MODULE__(), file; mode=:includet) + sleep(mtimedelay) + @eval import .$name + ExternalMT = @eval $name + foo, bar, mt = ExternalMT.foo, ExternalMT.bar, ExternalMT.method_table + sleep(mtimedelay) + @test foo() == 1 + @test bar() == 1 + (; ms) = Base.MethodList(mt) + @test retval(first(ms)) == 2 + rm(file) + sleep(mtimedelay) + write(file, second_revision(name)) + sleep(mtimedelay) + @yry() + @test foo() == 1 + @test bar() == 2 + (; ms) = Base.MethodList(mt) + @test retval(first(ms)) == 3 + end end do_test("Utilities") && @testset "Utilities" begin From e148172881d259168d27194ccb51a3a09e45d348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 13:37:57 -0400 Subject: [PATCH 02/17] Bump compat for CodeTracking --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 2677999f..d1298355 100644 --- a/Project.toml +++ b/Project.toml @@ -21,7 +21,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" DistributedExt = "Distributed" [compat] -CodeTracking = "1.2" +CodeTracking = "2" Distributed = "1" JuliaInterpreter = "0.9" LoweredCodeUtils = "3.2" From 5a38f9ca3a95b11d9e48c822b8e9c74fb3efdb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 14:44:58 -0400 Subject: [PATCH 03/17] Use MethodInfoKey, rename a few `sigs` to `mt_sigs` --- src/Revise.jl | 2 +- src/lowered.jl | 4 ++-- src/packagedef.jl | 49 ++++++++++++++++++++++++----------------------- src/types.jl | 2 +- test/runtests.jl | 5 +++-- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/Revise.jl b/src/Revise.jl index 78cacd8e..d904cb03 100644 --- a/src/Revise.jl +++ b/src/Revise.jl @@ -35,7 +35,7 @@ module Revise using OrderedCollections, CodeTracking, JuliaInterpreter, LoweredCodeUtils -using CodeTracking: PkgFiles, basedir, srcfiles, basepath +using CodeTracking: PkgFiles, basedir, srcfiles, basepath, MethodInfoKey using JuliaInterpreter: whichtt, is_doc_expr, step_expr!, finish_and_return!, get_return, @lookup, moduleof, scopeof, pc_expr, is_quotenode_egal, linetable, codelocs, LineTypes, isassign, isidentical diff --git a/src/lowered.jl b/src/lowered.jl index 28bcbb25..947d0123 100644 --- a/src/lowered.jl +++ b/src/lowered.jl @@ -16,7 +16,7 @@ end # This defines the API needed to store signatures using methods_by_execution! # This default version is simple and only used for testing purposes. # The "real" one is CodeTrackingMethodInfo in Revise.jl. -const MethodInfo = IdDict{Pair{<:Union{Nothing, MethodTable},<:Type},LineNumberNode} +const MethodInfo = IdDict{MethodInfoKey,LineNumberNode} add_signature!(methodinfo::MethodInfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln) push_expr!(methodinfo::MethodInfo, mod::Module, ex::Expr) = methodinfo pop_expr!(methodinfo::MethodInfo) = methodinfo @@ -320,7 +320,7 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra mod = moduleof(frame) # Hoist this lookup for performance. Don't throw even when `mod` is a baremodule: modinclude = isdefined(mod, :include) ? getfield(mod, :include) : nothing - signatures = [] # temporary for method signature storage + signatures = MethodInfoKey[] # temporary for method signature storage pc = frame.pc while true JuliaInterpreter.is_leaf(frame) || (@warn("not a leaf"); break) diff --git a/src/packagedef.jl b/src/packagedef.jl index f404c891..4dd0470d 100644 --- a/src/packagedef.jl +++ b/src/packagedef.jl @@ -282,18 +282,19 @@ get_method_from_match(mm::Core.MethodMatch) = mm.method function delete_missing!(exs_sigs_old::ExprsSigs, exs_sigs_new) with_logger(_debug_logger) do - for (ex, sigs) in exs_sigs_old + for (ex, mt_sigs) in exs_sigs_old haskey(exs_sigs_new, ex) && continue # ex was deleted - sigs === nothing && continue - for (mt, sig) in sigs + mt_sigs === nothing && continue + for mt_sig in mt_sigs + mt, sig = mt_sig ret = Base._methods_by_ftype(sig, mt, -1, Base.get_world_counter()) success = false if !isempty(ret) m = get_method_from_match(ret[end]) # the last method returned is the least-specific that matches, and thus most likely to be type-equal methsig = m.sig if sig <: methsig && methsig <: sig - locdefs = get(CodeTracking.method_info, mt => sig, nothing) + locdefs = get(CodeTracking.method_info, mt_sig, nothing) if isa(locdefs, Vector{Tuple{LineNumberNode,Expr}}) if length(locdefs) > 1 # Just delete this reference but keep the method @@ -319,7 +320,7 @@ function delete_missing!(exs_sigs_old::ExprsSigs, exs_sigs_new) end Base.delete_method(m) # Remove the entries from CodeTracking data - delete!(CodeTracking.method_info, mt => sig) + delete!(CodeTracking.method_info, mt_sig) # Remove frame from JuliaInterpreter, if applicable. Otherwise debuggers # may erroneously work with outdated code (265-like problems) if haskey(JuliaInterpreter.framedict, m) @@ -352,14 +353,14 @@ end function eval_rex(rex::RelocatableExpr, exs_sigs_old::ExprsSigs, mod::Module; mode::Symbol=:eval) return with_logger(_debug_logger) do - sigs, includes = nothing, nothing + mt_sigs, includes = nothing, nothing rexo = getkey(exs_sigs_old, rex, nothing) # extract the signatures and update the line info if rexo === nothing ex = rex.ex # ex is not present in old @debug "Eval" _group="Action" time=time() deltainfo=(mod, ex) - sigs, deps, includes, thunk = eval_with_signatures(mod, ex; mode=mode) # All signatures defined by `ex` + mt_sigs, deps, includes, thunk = eval_with_signatures(mod, ex; mode=mode) # All signatures defined by `ex` if !isexpr(thunk, :thunk) thunk = ex end @@ -376,14 +377,14 @@ function eval_rex(rex::RelocatableExpr, exs_sigs_old::ExprsSigs, mod::Module; mo end storedeps(deps, rex, mod) else - sigs = exs_sigs_old[rexo] + mt_sigs = exs_sigs_old[rexo] # Update location info ln, lno = firstline(unwrap(rex)), firstline(unwrap(rexo)) - if sigs !== nothing && !isempty(sigs) && ln != lno + if mt_sigs !== nothing && !isempty(mt_sigs) && ln != lno ln, lno = ln::LineNumberNode, lno::LineNumberNode - @debug "LineOffset" _group="Action" time=time() deltainfo=(sigs, lno=>ln) - for sig in sigs - locdefs = CodeTracking.method_info[sig]::AbstractVector + @debug "LineOffset" _group="Action" time=time() deltainfo=(mt_sigs, lno=>ln) + for mt_sig in mt_sigs + locdefs = CodeTracking.method_info[mt_sig]::AbstractVector ld = let lno=lno map(pr->linediff(lno, pr[1]), locdefs) end @@ -397,7 +398,7 @@ function eval_rex(rex::RelocatableExpr, exs_sigs_old::ExprsSigs, mod::Module; mo end end end - return sigs, includes + return mt_sigs, includes end end @@ -453,20 +454,20 @@ It also has the following fields: """ struct CodeTrackingMethodInfo exprstack::Vector{Expr} - allsigs::Vector{Any} + allsigs::Vector{Pair{Union{Nothing, MethodTable}, Type}} deps::Set{Union{GlobalRef,Symbol}} includes::Vector{Pair{Module,String}} end CodeTrackingMethodInfo(ex::Expr) = CodeTrackingMethodInfo([ex], Any[], Set{Union{GlobalRef,Symbol}}(), Pair{Module,String}[]) -function add_signature!(methodinfo::CodeTrackingMethodInfo, @nospecialize(sig), ln) - locdefs = CodeTracking.invoked_get!(Vector{Tuple{LineNumberNode,Expr}}, CodeTracking.method_info, sig) +function add_signature!(methodinfo::CodeTrackingMethodInfo, mt_sig::MethodInfoKey, ln) + locdefs = CodeTracking.invoked_get!(Vector{Tuple{LineNumberNode,Expr}}, CodeTracking.method_info, mt_sig) newdef = unwrap(methodinfo.exprstack[end]) if newdef !== nothing if !any(locdef->locdef[1] == ln && isequal(RelocatableExpr(locdef[2]), RelocatableExpr(newdef)), locdefs) push!(locdefs, (fixpath(ln), newdef)) end - push!(methodinfo.allsigs, sig) + push!(methodinfo.allsigs, mt_sig) end return methodinfo end @@ -1161,7 +1162,7 @@ function get_def(method::Method; modified_files=revision_queue) # We need to find the right file. if method.module == Base || method.module == Core || method.module == Core.Compiler @warn "skipping $method to avoid parsing too much code" - CodeTracking.invoked_setindex!(CodeTracking.method_info, CodeTracking.method_info_key(method), missing) + CodeTracking.invoked_setindex!(CodeTracking.method_info, missing, MethodInfoKey(method)) return false end parentfile, included_files = modulefiles(method.module) @@ -1180,13 +1181,13 @@ function get_def(method::Method; modified_files=revision_queue) end @warn "$(method.sig)$(isdefined(method, :external_mt) ? " (overlayed)" : "") was not found" # So that we don't call it again, store missingness info in CodeTracking - CodeTracking.invoked_setindex!(CodeTracking.method_info, CodeTracking.method_info_key(method), missing) + CodeTracking.invoked_setindex!(CodeTracking.method_info, missing, MethodInfoKey(method)) return false end function get_def(method, pkgdata, filename) maybe_extract_sigs!(maybe_parse_from_cache!(pkgdata, filename)) - return get(CodeTracking.method_info, CodeTracking.method_info_key(method), nothing) + return get(CodeTracking.method_info, MethodInfoKey(method), nothing) end function get_tracked_id(id::PkgId; modified_files=revision_queue) @@ -1247,7 +1248,7 @@ function update_stacktrace_lineno!(trace) m = linfo.def # Why not just call `whereis`? Because that forces tracking. This is being # clever by recognizing that these entries exist only if there have been updates. - updated = get(CodeTracking.method_info, CodeTracking.method_info_key(m), nothing) + updated = get(CodeTracking.method_info, MethodInfoKey(m), nothing) if updated !== nothing lnn = updated[1][1] # choose the first entry by default lineoffset = lnn.line - m.line @@ -1262,7 +1263,7 @@ end function method_location(method::Method) # Why not just call `whereis`? Because that forces tracking. This is being # clever by recognizing that these entries exist only if there have been updates. - updated = get(CodeTracking.method_info, CodeTracking.method_info_key(method), nothing) + updated = get(CodeTracking.method_info, MethodInfoKey(method), nothing) if updated !== nothing lnn = updated[1][1] return lnn.file, lnn.line @@ -1320,7 +1321,7 @@ Revise itself does not need to be running on `p`. """ function init_worker(p::AbstractWorker) @invokelatest remotecall_impl(Core.eval, p, Main, quote - function whichtt(mt::Union{Nothing, MethodTable}, @nospecialize sig) + function whichtt(mt::Union{Nothing, Core.MethodTable}, @nospecialize sig) ret = Base._methods_by_ftype(sig, mt, -1, Base.get_world_counter()) isempty(ret) && return nothing m = ret[end][3]::Method # the last method returned is the least-specific that matches, and thus most likely to be type-equal @@ -1328,7 +1329,7 @@ function init_worker(p::AbstractWorker) (sig <: methsig && methsig <: sig) || return nothing return m end - function delete_method_by_sig(mt::Union{Nothing, MethodTable}, @nospecialize sig) + function delete_method_by_sig(mt::Union{Nothing, Core.MethodTable}, @nospecialize sig) m = whichtt(mt, sig) isa(m, Method) && Base.delete_method(m) end diff --git a/src/types.jl b/src/types.jl index 011f0a7c..5ad7f2f7 100644 --- a/src/types.jl +++ b/src/types.jl @@ -16,7 +16,7 @@ mutable struct WatchList end const DocExprs = Dict{Module,Vector{Expr}} -const ExprsSigs = OrderedDict{RelocatableExpr,Union{Nothing,Vector{Any}}} +const ExprsSigs = OrderedDict{RelocatableExpr,Union{Nothing,Vector{Pair{Union{Nothing, MethodTable}, Type}}}} const DepDictVals = Tuple{Module,RelocatableExpr} const DepDict = Dict{Symbol,Set{DepDictVals}} diff --git a/test/runtests.jl b/test/runtests.jl index 589e4b0f..8119c733 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ # REVISE: DO NOT PARSE # For people with JULIA_REVISE_INCLUDE=1 using Revise using Revise.CodeTracking +using Revise.CodeTracking: MethodInfoKey using Revise.JuliaInterpreter using Test @@ -1408,7 +1409,7 @@ end """) @yry() @test MacroSigs.blah() == 1 - @test haskey(CodeTracking.method_info, CodeTracking.method_info_key(@which MacroSigs.blah())) + @test haskey(CodeTracking.method_info, MethodInfoKey(@which MacroSigs.blah())) rm_precompile("MacroSigs") # Issue #568 (a macro *execution* bug) @@ -2945,7 +2946,7 @@ end do_test("Recipes") && @testset "Recipes" begin # https://github.com/JunoLab/Juno.jl/issues/257#issuecomment-473856452 meth = @which gcd(10, 20) - sigs = signatures_at(Base.find_source_file(String(meth.file)), meth.line) # this should track Base + signatures_at(Base.find_source_file(String(meth.file)), meth.line) # this should track Base # Tracking Base # issue #250 From 9cac5356d4258200dd8c5e95ca855889ee9a0cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 14:49:45 -0400 Subject: [PATCH 04/17] Start updating docs a bit --- docs/src/internals.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 3ff9388b..76422c9c 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -226,9 +226,9 @@ Most of Revise's magic comes down to just three internal variables: Two "maps" are central to Revise's inner workings: `ExprsSigs` maps link definition=>signature-types (the forward workflow), while `CodeTracking` (specifically, -its internal variable `method_info`) links from -signature-type=>definition (the backward workflow). -Concretely, `CodeTracking.method_info` is just an `IdDict` mapping `sigt=>(locationinfo, def)`. +its internal variable `method_info`) links from a +method table/signature-type pair to the corresponding definition (the backward workflow). +Concretely, `CodeTracking.method_info` is just an `IdDict` mapping `(mt => sigt) => (locationinfo, def)`. Of note, a stack frame typically contains a link to a method, which stores the equivalent of `sigt`; consequently, this information allows one to look up the corresponding `locationinfo` and `def`. (When methods move, the location information stored by CodeTracking @@ -237,8 +237,8 @@ gets updated by Revise.) Some additional notes about Revise's `ExprsSigs` maps: - For expressions that do not define a method, it is just `def=>nothing` -- For expressions that do define a method, it is `def=>[sigt1, ...]`. - `[sigt1, ...]` is the list of signature-types generated from `def` (often just one, +- For expressions that do define a method, it is `def=>[mt_sigt1, ...]`. + `[mt_sigt1, ...]` is the list of method table/signature-type pairs generated from `def` (often just one, but more in the case of methods with default arguments or keyword arguments). - They are represented as an `OrderedDict` so as to preserve the sequence in which expressions occur in the file. @@ -255,7 +255,7 @@ Some additional notes about Revise's `ExprsSigs` maps: the location information stored by `CodeTracking`. `ExprsSigs` are organized by module and then file, so that one can map -`filename`=>`module`=>`def`=>`sigts`. +`filename`=>`module`=>`def`=>`mt_sigts`. Importantly, single-file modules can be "reconstructed" from the keys of the corresponding `ExprsSigs` (and multi-file modules from a collection of such items), since they hold the complete ordered set of expressions that would be `eval`ed to define the module. @@ -334,13 +334,13 @@ FileInfo(Items=>ExprsSigs with the following expressions: end), ) ``` -This is just a summary; to see the actual `def=>sigts` map, do the following: +This is just a summary; to see the actual `def=>mt_sigts` map, do the following: ```julia julia> pkgdata.fileinfos[2].modexsigs[Items] -OrderedCollections.OrderedDict{Revise.RelocatableExpr,Union{Nothing, Array{Any,1}}} with 2 entries: - :(indent(::UInt16) = begin… => Any[Tuple{typeof(indent),UInt16}] - :(indent(::UInt8) = begin… => Any[Tuple{typeof(indent),UInt8}] +OrderedCollections.OrderedDict{Module, OrderedCollections.OrderedDict{Revise.RelocatableExpr, Union{Nothing, Vector{Pair{Union{Nothing, Core.MethodTable}, Type}}}}} with 2 entries: + :(indent(::UInt16) = begin… => Pair{Union{Nothing, MethodTable}, Type}[nothing => Tuple{typeof(indent),UInt16}] + :(indent(::UInt8) = begin… => Pair{Union{Nothing, MethodTable}, Type}[nothing => Tuple{typeof(indent),UInt8}] ``` These are populated now because we specified `__precompile__(false)`, which forces From fe7acd4a1f22e90e1626008bc5b6adabd653d498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 14:52:28 -0400 Subject: [PATCH 05/17] Patch CI to test with upstream dependencies --- .github/workflows/Documenter.yml | 2 ++ .github/workflows/ci.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index b91b123b..2b5ed12b 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' + - run: cd docs && julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-docdeploy@latest env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 476a13c7..e5aaf05c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: version: ${{ matrix.version }} show-versioninfo: ${{ matrix.version == 'nightly' }} - uses: julia-actions/cache@v2 + - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' - uses: julia-actions/julia-buildpkg@latest # Revise's tests need significant customization # Populate the precompile cache with an extraneous file, to catch issues like in #460 From 27d82f7beeaea3563477855c904e08d9492f2e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 14:59:50 -0400 Subject: [PATCH 06/17] Update outdated comment --- src/lowered.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lowered.jl b/src/lowered.jl index 947d0123..f4c85d11 100644 --- a/src/lowered.jl +++ b/src/lowered.jl @@ -15,7 +15,7 @@ end # This defines the API needed to store signatures using methods_by_execution! # This default version is simple and only used for testing purposes. -# The "real" one is CodeTrackingMethodInfo in Revise.jl. +# The "real" one is CodeTrackingMethodInfo in packagedef.jl. const MethodInfo = IdDict{MethodInfoKey,LineNumberNode} add_signature!(methodinfo::MethodInfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln) push_expr!(methodinfo::MethodInfo, mod::Module, ex::Expr) = methodinfo From da595896f79bab3c0961f8e0f12e7724c90f9786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Mon, 21 Apr 2025 15:23:55 -0400 Subject: [PATCH 07/17] Attempt to fix Git tests --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5aaf05c..0ff5491d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,7 @@ jobs: # We also need to pick up the Git tests, but for that we need to `dev` the package echo "Git tests" julia --code-coverage=user -e ' + using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables" using Pkg; Pkg.develop(PackageSpec(path=".")) include(joinpath("test", "runtests.jl")) ' "Git" From 1c03d7d31a2c2ee7ab33afcace30cfc38540a14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 24 Apr 2025 12:36:30 -0400 Subject: [PATCH 08/17] Refactor tests and increase coverage --- test/runtests.jl | 79 +++++++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8119c733..f2ce9a8c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3100,7 +3100,8 @@ end do_test("External method tables") && @testset "External method tables" begin function retval(m) src = Base.uncompressed_ast(m) - node = src.code[1]::Core.ReturnNode + i = findfirst(!isnothing, src.code) + node = src.code[i]::Core.ReturnNode node.val end @@ -3111,9 +3112,20 @@ end module $name Base.Experimental.@MethodTable(method_table) + Base.Experimental.@MethodTable(method_table_2) + get_method_table() = method_table + + macro override(ex) esc(:(Base.Experimental.@overlay \$method_table \$ex)) end + macro override_2(ex) esc(:(Base.Experimental.@overlay $name.method_table \$ex)) end + macro override_3(ex) esc(:(Base.Experimental.@overlay get_method_table() \$ex)) end foo() = 1 - Base.Experimental.@overlay method_table foo() = 2 + + @override print(x) = "print" + @override cos(x) = "cos" + @override_2 sin(x) = "sin" + @override_3 sincos(x) = "sincos" + Base.Experimental.@overlay method_table_2 foo() = 2 bar() = foo() @@ -3123,35 +3135,59 @@ end module $name Base.Experimental.@MethodTable(method_table) + Base.Experimental.@MethodTable(method_table_2) + get_method_table() = method_table + + macro override(ex) esc(:(Base.Experimental.@overlay \$method_table \$ex)) end + macro override_2(ex) esc(:(Base.Experimental.@overlay $name.method_table \$ex)) end + macro override_3(ex) esc(:(Base.Experimental.@overlay get_method_table() \$ex)) end foo() = 1 - Base.Experimental.@overlay method_table foo() = 3 + + @override print(x) = "print" + @override cos(x) = "sin" + @override_2 sin(x) = "cos" + @override_3 sincos(x) = "cossin" + Base.Experimental.@overlay method_table_2 foo() = 3 bar() = foo() + 1 end """ + function test_first_revision(mod) + foo, bar, mt, mt2 = mod.foo, mod.bar, mod.method_table, mod.method_table_2 + @test foo() == 1 + @test bar() == 1 + (; ms) = Base.MethodList(mt) + @test length(ms) == 4 # cos/sin/sincos/print + (; ms) = Base.MethodList(mt2) + @test length(ms) == 1 # foo + @test retval(first(ms)) == 2 + end + + function test_second_revision(mod) + foo, bar, mt, mt2 = mod.foo, mod.bar, mod.method_table, mod.method_table_2 + @test foo() == 1 + @test bar() == 2 + (; ms) = Base.MethodList(mt) + @test length(ms) == 7 # cos/sin/sincos x2 + print x1 + (; ms) = Base.MethodList(mt2) + @test length(ms) == 2 # foo x2 + @test retval(first(ms)) == 3 + end + name = unique_name(:ExternalMT) dn = joinpath(testdir, "$name", "src") mkpath(dn) write(joinpath(dn, "$name.jl"), first_revision(name)) sleep(mtimedelay) @eval import $name - ExternalMT = @eval $name - foo, bar, mt = ExternalMT.foo, ExternalMT.bar, ExternalMT.method_table sleep(mtimedelay) - @test foo() == 1 - @test bar() == 1 - (; ms) = Base.MethodList(mt) - @test retval(first(ms)) == 2 + test_first_revision(@eval $name) write(joinpath(dn, "$name.jl"), second_revision(name)) - sleep(mtimedelay) @yry() - @test foo() == 1 - @test bar() == 2 - (; ms) = Base.MethodList(mt) - @test retval(first(ms)) == 3 + test_second_revision(@eval $name) file = tempname() * ".jl" name = unique_name(:ExternalMT_includet) @@ -3160,22 +3196,11 @@ end Revise.track(@__MODULE__(), file; mode=:includet) sleep(mtimedelay) @eval import .$name - ExternalMT = @eval $name - foo, bar, mt = ExternalMT.foo, ExternalMT.bar, ExternalMT.method_table - sleep(mtimedelay) - @test foo() == 1 - @test bar() == 1 - (; ms) = Base.MethodList(mt) - @test retval(first(ms)) == 2 - rm(file) sleep(mtimedelay) + test_first_revision(@eval $name) write(file, second_revision(name)) - sleep(mtimedelay) @yry() - @test foo() == 1 - @test bar() == 2 - (; ms) = Base.MethodList(mt) - @test retval(first(ms)) == 3 + test_second_revision(@eval $name) end end From 83c5eb9cd9e259189e5cfb1785c11a44e774a3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 24 Apr 2025 14:00:02 -0400 Subject: [PATCH 09/17] Add tests for method deletion --- test/runtests.jl | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f2ce9a8c..e2ddf1ca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3122,12 +3122,14 @@ end foo() = 1 @override print(x) = "print" + @override show(x) = "show" @override cos(x) = "cos" @override_2 sin(x) = "sin" @override_3 sincos(x) = "sincos" Base.Experimental.@overlay method_table_2 foo() = 2 bar() = foo() + baz() = bar() end """ @@ -3145,35 +3147,39 @@ end foo() = 1 @override print(x) = "print" + # @override show(x) = "show" @override cos(x) = "sin" @override_2 sin(x) = "cos" @override_3 sincos(x) = "cossin" Base.Experimental.@overlay method_table_2 foo() = 3 bar() = foo() + 1 + # baz() = bar() end """ - function test_first_revision(mod) - foo, bar, mt, mt2 = mod.foo, mod.bar, mod.method_table, mod.method_table_2 - @test foo() == 1 - @test bar() == 1 - (; ms) = Base.MethodList(mt) - @test length(ms) == 4 # cos/sin/sincos/print - (; ms) = Base.MethodList(mt2) + function test_first_revision(mod::Module) + @test mod.foo() == 1 + @test mod.bar() == 1 + @test length(methods(mod.baz)) == 1 + (; ms) = Base.MethodList(mod.method_table) + @test length(ms) == 5 # cos/sin/sincos/print/show + (; ms) = Base.MethodList(mod.method_table_2) @test length(ms) == 1 # foo @test retval(first(ms)) == 2 end - function test_second_revision(mod) - foo, bar, mt, mt2 = mod.foo, mod.bar, mod.method_table, mod.method_table_2 - @test foo() == 1 - @test bar() == 2 - (; ms) = Base.MethodList(mt) - @test length(ms) == 7 # cos/sin/sincos x2 + print x1 - (; ms) = Base.MethodList(mt2) + function test_second_revision(mod::Module) + @test mod.foo() == 1 + @test mod.bar() == 2 + @test isempty(methods(mod.baz)) + (; ms) = Base.MethodList(mod.method_table) + @test length(ms) == 8 # cos/sin/sincos x2 + print/show + @test count(x -> x.deleted_world < Base.tls_world_age(), ms) == 4 # deleted cos/sin/sincos/show + (; ms) = Base.MethodList(mod.method_table_2) @test length(ms) == 2 # foo x2 + @test count(x -> x.deleted_world < Base.tls_world_age(), ms) == 1 # deleted foo @test retval(first(ms)) == 3 end From bb750c95c262e969c07193b193512cdbb3e05b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 24 Apr 2025 14:18:10 -0400 Subject: [PATCH 10/17] Bump JuliaInterpreter compat bound --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index d1298355..9dd26315 100644 --- a/Project.toml +++ b/Project.toml @@ -23,7 +23,7 @@ DistributedExt = "Distributed" [compat] CodeTracking = "2" Distributed = "1" -JuliaInterpreter = "0.9" +JuliaInterpreter = "0.10" LoweredCodeUtils = "3.2" OrderedCollections = "1" # Exclude Requires-1.1.0 - see https://github.com/JuliaPackaging/Requires.jl/issues/94 From b90f43044395309b41e306047ccbca66dc1c6712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Thu, 24 Apr 2025 16:59:24 -0400 Subject: [PATCH 11/17] Fix tests for 1.10 --- test/runtests.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 7db7ea78..8837ce45 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3171,15 +3171,16 @@ end end function test_second_revision(mod::Module) + current_world_age = isdefined(Base, :tls_world_age) ? Base.tls_world_age() : Base.get_world_counter() @test mod.foo() == 1 @test mod.bar() == 2 @test isempty(methods(mod.baz)) (; ms) = Base.MethodList(mod.method_table) @test length(ms) == 8 # cos/sin/sincos x2 + print/show - @test count(x -> x.deleted_world < Base.tls_world_age(), ms) == 4 # deleted cos/sin/sincos/show + @test count(x -> x.deleted_world < current_world_age, ms) == 4 # deleted cos/sin/sincos/show (; ms) = Base.MethodList(mod.method_table_2) @test length(ms) == 2 # foo x2 - @test count(x -> x.deleted_world < Base.tls_world_age(), ms) == 1 # deleted foo + @test count(x -> x.deleted_world < current_world_age, ms) == 1 # deleted foo @test retval(first(ms)) == 3 end From a20db58ed6ec320a85ad785ec08468bdce1ca574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 25 Apr 2025 13:27:49 -0400 Subject: [PATCH 12/17] Update documentation, more `mt_sigs` renames --- docs/src/debugging.md | 6 +++--- docs/src/internals.md | 1 + src/packagedef.jl | 22 +++++++++++----------- src/types.jl | 18 +++++++++--------- test/sigtest.jl | 4 ++-- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/src/debugging.md b/docs/src/debugging.md index 93dcc9c7..e373f26d 100644 --- a/docs/src/debugging.md +++ b/docs/src/debugging.md @@ -159,13 +159,13 @@ julia> rlogger.logs #= /tmp/revisetest.jl:9 =# x ^ 4 end)))) - Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459331061e9, deltainfo=(Any[Tuple{typeof(mult2),Any}], :(#= /tmp/revisetest.jl:11 =#) => :(#= /tmp/revisetest.jl:13 =#)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459331061e9, deltainfo=(Pair{Union{Nothing, MethodTable}, Type}[nothing => Tuple{typeof(mult2),Any}], :(#= /tmp/revisetest.jl:11 =#) => :(#= /tmp/revisetest.jl:13 =#)))) Revise.LogRecord(Debug, Eval, Action, Revise_9147188b, "/home/tim/.julia/dev/Revise/src/Revise.jl", 276, (time=1.557996459391182e9, deltainfo=(Main.ReviseTest.Internal, :(mult3(x) = begin #= /tmp/revisetest.jl:14 =# 3x end)))) - Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391642e9, deltainfo=(Any[Tuple{typeof(unchanged),Any}], :(#= /tmp/revisetest.jl:18 =#) => :(#= /tmp/revisetest.jl:19 =#)))) - Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391695e9, deltainfo=(Any[Tuple{typeof(unchanged2),Any}], :(#= /tmp/revisetest.jl:20 =#) => :(#= /tmp/revisetest.jl:21 =#)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391642e9, deltainfo=(Pair{Union{Nothing, MethodTable}, Type}[nothing => Tuple{typeof(unchanged),Any}], :(#= /tmp/revisetest.jl:18 =#) => :(#= /tmp/revisetest.jl:19 =#)))) + Revise.LogRecord(Debug, LineOffset, Action, Revise_fb38a7f7, "/home/tim/.julia/dev/Revise/src/Revise.jl", 296, (time=1.557996459391695e9, deltainfo=(Pair{Union{Nothing, MethodTable}, Type}[nothing => Tuple{typeof(unchanged2),Any}], :(#= /tmp/revisetest.jl:20 =#) => :(#= /tmp/revisetest.jl:21 =#)))) ``` You can see that Revise started by deleting three methods, followed by evaluating three new versions of those methods. Interspersed are various changes to the line numbering. diff --git a/docs/src/internals.md b/docs/src/internals.md index 76422c9c..4406f155 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -137,6 +137,7 @@ Tuple{typeof(print_item),IO,Any,Integer,String} # print_item(io, item, 2, " ``` In Revise's internal code, a definition is often represented with a variable `def`, and a signature-type with `sigt`. +The method table for which the method was defined is also represented, to form a `mt_sigt` pair. Recent versions of Revise do not make extensive use of signature expressions. ### Computing signatures diff --git a/src/packagedef.jl b/src/packagedef.jl index b7c72f52..51e971fe 100644 --- a/src/packagedef.jl +++ b/src/packagedef.jl @@ -264,7 +264,7 @@ const silencefile = Ref(joinpath(depsdir, "silence.txt")) # Ref so that tests d ## + add to the ModuleExprsSigs ## + add to CodeTracking.method_info ## -## Interestingly, the ex=>sigs link may not be the same as the sigs=>ex link. +## Interestingly, the ex=>mt_sigs link may not be the same as the mt_sigs=>ex link. ## Consider a conditional block, ## if Sys.islinux() ## f() = 1 @@ -407,9 +407,9 @@ end function eval_new!(exs_sigs_new::ExprsSigs, exs_sigs_old, mod::Module; mode::Symbol=:eval) includes = Vector{Pair{Module,String}}() for rex in keys(exs_sigs_new) - sigs, _includes = eval_rex(rex, exs_sigs_old, mod; mode=mode) - if sigs !== nothing - exs_sigs_new[rex] = sigs + mt_sigs, _includes = eval_rex(rex, exs_sigs_old, mod; mode=mode) + if mt_sigs !== nothing + exs_sigs_new[rex] = mt_sigs end if _includes !== nothing append!(includes, _includes) @@ -524,8 +524,8 @@ function instantiate_sigs!(modexsigs::ModuleExprsSigs; mode=:sigs, kwargs...) for (mod, exsigs) in modexsigs for rex in keys(exsigs) is_doc_expr(rex.ex) && continue - sigs, deps, _ = eval_with_signatures(mod, rex.ex; mode=mode, kwargs...) - exsigs[rex] = sigs + mt_sigs, deps, _ = eval_with_signatures(mod, rex.ex; mode=mode, kwargs...) + exsigs[rex] = mt_sigs storedeps(deps, rex, mod) end end @@ -848,9 +848,9 @@ function revise(; throw=false) mode ∈ (:sigs, :eval, :evalmeth, :evalassign) || error("unsupported mode ", mode) exsold = get(fi.modexsigs, mod, empty_exs_sigs) for rex in keys(exsnew) - sigs, includes = eval_rex(rex, exsold, mod; mode=mode) - if sigs !== nothing - exsnew[rex] = sigs + mt_sigs, includes = eval_rex(rex, exsold, mod; mode=mode) + if mt_sigs !== nothing + exsnew[rex] = mt_sigs end if includes !== nothing maybe_add_includes_to_pkgdata!(pkgdata, file, includes; eval_now=true) @@ -1144,8 +1144,8 @@ function get_def(method::Method; modified_files=revision_queue) fi = add_definitions_from_repl(filename) hassig = false for (mod, exs) in fi.modexsigs - for sigs in values(exs) - hassig |= !isempty(sigs) + for mt_sigs in values(exs) + hassig |= !isempty(mt_sigs) end end return hassig diff --git a/src/types.jl b/src/types.jl index 5ad7f2f7..cf111aa3 100644 --- a/src/types.jl +++ b/src/types.jl @@ -24,9 +24,9 @@ function Base.show(io::IO, exsigs::ExprsSigs) compact = get(io, :compact, false) if compact n = 0 - for (rex, sigs) in exsigs - sigs === nothing && continue - n += length(sigs) + for (rex, mt_sigs) in exsigs + mt_sigs === nothing && continue + n += length(mt_sigs) end print(io, "ExprsSigs(<$(length(exsigs)) expressions>, <$n signatures>)") else @@ -42,11 +42,11 @@ end ModuleExprsSigs For a particular source file, the corresponding `ModuleExprsSigs` is a mapping -`mod=>exprs=>sigs` of the expressions `exprs` found in `mod` and the signatures `sigs` +`mod=>exprs=>mt_sigs` of the expressions `exprs` found in `mod` and the method table/signature pairs `mt_sigs` that arise from them. Specifically, if `mes` is a `ModuleExprsSigs`, then `mes[mod][ex]` -is a list of signatures that result from evaluating `ex` in `mod`. It is possible that +is a list of method table/signature pairs that result from evaluating `ex` in `mod`. It is possible that this returns `nothing`, which can mean either that `ex` does not define any methods -or that the signatures have not yet been cached. +or that the method table/signature pairs have not yet been cached. The first `mod` key is guaranteed to be the module into which this file was `include`d. @@ -186,10 +186,10 @@ function Base.show(io::IO, pkgdata::PkgData) for fi in pkgdata.fileinfos thisnexs, thisnsigs = 0, 0 for (mod, exsigs) in fi.modexsigs - for (rex, sigs) in exsigs + for (rex, mt_sigs) in exsigs thisnexs += 1 - sigs === nothing && continue - thisnsigs += length(sigs) + mt_sigs === nothing && continue + thisnsigs += length(mt_sigs) end end nexs += thisnexs diff --git a/test/sigtest.jl b/test/sigtest.jl index 500d9360..d7609337 100644 --- a/test/sigtest.jl +++ b/test/sigtest.jl @@ -88,8 +88,8 @@ module Lowering end end end end - sigs, _ = Revise.eval_with_signatures(Lowering, ex) - @test length(sigs) >= 2 + mt_sigs, _ = Revise.eval_with_signatures(Lowering, ex) + @test length(mt_sigs) >= 2 end try # Suppress world age increments, since the instantiation messes with base From 4f92e45186f7eb5a3975b1c93d98a2f7c007bd1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 25 Apr 2025 15:37:07 -0400 Subject: [PATCH 13/17] Add NEWS.md entry --- NEWS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NEWS.md b/NEWS.md index 0fe8bf2d..ef25c1ae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,10 @@ This file describes only major changes, and does not include bug fixes, cleanups, or minor enhancements. +## Revise 3.8 + +* Methods defined on external method tables via `Base.Experimental.@overlay` can now be correctly revised ([#904]). + ## Revise 3.3 * Upgrade to JuliaInterpreter 0.9 and drop support for Julia prior to 1.6 (the new LTS). @@ -151,3 +155,4 @@ New features: [#243]: https://github.com/timholy/Revise.jl/pull/243 [#245]: https://github.com/timholy/Revise.jl/pull/245 [#278]: https://github.com/timholy/Revise.jl/pull/278 +[#904]: https://github.com/timholy/Revise.jl/pull/904 From f3a9e9983d93960eeef58623290d3230046483e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 25 Apr 2025 15:37:19 -0400 Subject: [PATCH 14/17] Bump package version to 3.8.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 1f44822c..35c72b3f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Revise" uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" -version = "3.7.6" +version = "3.8.0" [deps] CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" From 024f45cab45c2673bc2f2dde4a96ee2688e024ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Belmant?= Date: Fri, 25 Apr 2025 15:43:10 -0400 Subject: [PATCH 15/17] Update CI patches --- .github/workflows/Documenter.yml | 4 ++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index 2b5ed12b..53d719d5 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -11,8 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' - - run: cd docs && julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' + - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' + - run: cd docs && julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-docdeploy@latest env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ff5491d..4db9539e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: version: ${{ matrix.version }} show-versioninfo: ${{ matrix.version == 'nightly' }} - uses: julia-actions/cache@v2 - - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' + - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' - uses: julia-actions/julia-buildpkg@latest # Revise's tests need significant customization # Populate the precompile cache with an extraneous file, to catch issues like in #460 @@ -81,7 +81,7 @@ jobs: # We also need to pick up the Git tests, but for that we need to `dev` the package echo "Git tests" julia --code-coverage=user -e ' - using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/serenity4/CodeTracking.jl#parametrize-by-mt https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables" + using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables" using Pkg; Pkg.develop(PackageSpec(path=".")) include(joinpath("test", "runtests.jl")) ' "Git" From d3024453622a41677a53c388077d320a046ba6f0 Mon Sep 17 00:00:00 2001 From: serenity4 Date: Tue, 10 Jun 2025 15:27:35 +0200 Subject: [PATCH 16/17] Update CI dependency patch --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4db9539e..d4006aed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: version: ${{ matrix.version }} show-versioninfo: ${{ matrix.version == 'nightly' }} - uses: julia-actions/cache@v2 - - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables"' + - run: julia --project -e 'using Pkg; using Pkg; Pkg.add([PackageSpec(; url="https://github.com/serenity4/JuliaInterpreter.jl", rev="codetracking-v2"), PackageSpec(; name = "CodeTracking", rev="master"), PackageSpec(; url = "https://github.com/serenity4/LoweredCodeUtils.jl", rev = "support-external-methodtables")])' - uses: julia-actions/julia-buildpkg@latest # Revise's tests need significant customization # Populate the precompile cache with an extraneous file, to catch issues like in #460 @@ -81,7 +81,7 @@ jobs: # We also need to pick up the Git tests, but for that we need to `dev` the package echo "Git tests" julia --code-coverage=user -e ' - using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 https://github.com/timholy/CodeTracking.jl#master https://github.com/serenity4/LoweredCodeUtils.jl#support-external-methodtables" + using Pkg; using Pkg; using Pkg; Pkg.add([PackageSpec(; url="https://github.com/serenity4/JuliaInterpreter.jl", rev="codetracking-v2"), PackageSpec(; name = "CodeTracking", rev="master"), PackageSpec(; url = "https://github.com/serenity4/LoweredCodeUtils.jl", rev = "support-external-methodtables")]) using Pkg; Pkg.develop(PackageSpec(path=".")) include(joinpath("test", "runtests.jl")) ' "Git" From 13b67a18406fdf54d89b0ff11f3e604fb8ff593e Mon Sep 17 00:00:00 2001 From: serenity4 Date: Tue, 10 Jun 2025 15:51:39 +0200 Subject: [PATCH 17/17] Don't test `(::Method).deleted_world` on 1.12+ --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 68453776..fac401c6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3165,10 +3165,10 @@ const issue639report = [] @test isempty(methods(mod.baz)) (; ms) = Base.MethodList(mod.method_table) @test length(ms) == 8 # cos/sin/sincos x2 + print/show - @test count(x -> x.deleted_world < current_world_age, ms) == 4 # deleted cos/sin/sincos/show + VERSION < v"1.12-" && @test count(x -> x.deleted_world < current_world_age, ms) == 4 # deleted cos/sin/sincos/show (; ms) = Base.MethodList(mod.method_table_2) @test length(ms) == 2 # foo x2 - @test count(x -> x.deleted_world < current_world_age, ms) == 1 # deleted foo + VERSION < v"1.12-" && @test count(x -> x.deleted_world < current_world_age, ms) == 1 # deleted foo @test retval(first(ms)) == 3 end