diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 811de4d..cf14380 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -54,6 +54,7 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- + - run: julia --project -e 'using Pkg; Pkg.add([PackageSpec(; url="https://github.com/serenity4/JuliaInterpreter.jl", rev="codetracking-v2"), PackageSpec(; name = "CodeTracking", rev="master")])' - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index b91b123..3397e84 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - run: julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 CodeTracking#master"' + - run: cd docs && julia --project -e 'using Pkg; pkg"add https://github.com/serenity4/JuliaInterpreter.jl#codetracking-v2 CodeTracking#master"' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-docdeploy@latest env: diff --git a/Project.toml b/Project.toml index 264285e..aea8f50 100644 --- a/Project.toml +++ b/Project.toml @@ -4,9 +4,11 @@ version = "3.3.0" authors = ["Tim Holy "] [deps] +CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" JuliaInterpreter = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" [compat] +CodeTracking = "2" JuliaInterpreter = "0.10" julia = "1.10" diff --git a/src/LoweredCodeUtils.jl b/src/LoweredCodeUtils.jl index ef33ec5..8c87b72 100644 --- a/src/LoweredCodeUtils.jl +++ b/src/LoweredCodeUtils.jl @@ -9,11 +9,13 @@ module LoweredCodeUtils # This somewhat unusual structure is in place to support # the VS Code extension integration. +using CodeTracking: MethodInfoKey + using JuliaInterpreter using JuliaInterpreter: SSAValue, SlotNumber, Frame, Interpreter, RecursiveInterpreter using JuliaInterpreter: codelocation, is_global_ref, is_global_ref_egal, is_quotenode_egal, is_return, lookup, lookup_return, linetable, moduleof, next_until!, nstatements, pc_expr, - step_expr!, whichtt + step_expr!, whichtt, extract_method_table include("packagedef.jl") diff --git a/src/codeedges.jl b/src/codeedges.jl index 5e63b29..5608a2e 100644 --- a/src/codeedges.jl +++ b/src/codeedges.jl @@ -242,27 +242,25 @@ function direct_links!(cl::CodeLinks, src::CodeInfo) add_inner!(cl, icl, i) continue elseif isexpr(stmt, :method) - if length(stmt.args) === 3 && (arg3 = stmt.args[3]; arg3 isa CodeInfo) - icl = CodeLinks(cl.thismod, arg3) - add_inner!(cl, icl, i) - end - name = stmt.args[1] - if isa(name, GlobalRef) || isa(name, Symbol) + if length(stmt.args) === 1 + # A function with no methods was defined. Associate its new binding to it. + name = stmt.args[1] if isa(name, Symbol) name = GlobalRef(cl.thismod, name) end - assign = get(cl.nameassigns, name, nothing) - if assign === nothing - cl.nameassigns[name] = assign = Int[] + if !isa(name, GlobalRef) + @show stmt + error("name ", typeof(name), " not recognized") end + assign = get!(Vector{Int}, cl.nameassigns, name) push!(assign, i) targetstore = get!(Links, cl.namepreds, name) target = P(name, targetstore) add_links!(target, stmt, cl) - elseif name in (nothing, false) - else - @show stmt - error("name ", typeof(name), " not recognized") + elseif length(stmt.args) === 3 && (arg3 = stmt.args[3]; arg3 isa CodeInfo) # method definition + # A method was defined for an existing function. + icl = CodeLinks(cl.thismod, arg3) + add_inner!(cl, icl, i) end rhs = stmt target = P(SSAValue(i), cl.ssapreds[i]) diff --git a/src/packagedef.jl b/src/packagedef.jl index c99cedd..b25c166 100644 --- a/src/packagedef.jl +++ b/src/packagedef.jl @@ -1,6 +1,6 @@ Base.Experimental.@optlevel 1 -using Core: SimpleVector +using Core: SimpleVector, MethodTable using Core.IR: CodeInfo, GotoIfNot, GotoNode, IR, MethodInstance, ReturnNode @static if isdefined(Core.IR, :EnterNode) using Core.IR: EnterNode diff --git a/src/signatures.jl b/src/signatures.jl index e221633..3a3378c 100644 --- a/src/signatures.jl +++ b/src/signatures.jl @@ -24,9 +24,9 @@ function signature(sigsv::SimpleVector) end """ - sigt, lastpc = signature([interp::Interpreter=RecursiveInterpreter()], frame::Frame, pc::Int) + (mt, sigt), lastpc = signature([interp::Interpreter=RecursiveInterpreter()], frame::Frame, pc::Int) -Compute the signature-type `sigt` of a method whose definition in `frame` starts at `pc`. +Compute the method table `mt` and signature-type `sigt` of a method whose definition in `frame` starts at `pc`. Generally, `pc` should point to the `Expr(:method, methname)` statement, in which case `lastpc` is the final statement number in `frame` that is part of the signature (i.e, the line above the 3-argument `:method` expression). @@ -34,7 +34,7 @@ Alternatively, `pc` can point to the 3-argument `:method` expression, as long as all the relevant SSAValues have been assigned. In this case, `lastpc == pc`. -If no 3-argument `:method` expression is found, `sigt` will be `nothing`. +If no 3-argument `:method` expression is found, `nothing` will be returned in place of `(mt, sigt)`. """ function signature(interp::Interpreter, frame::Frame, @nospecialize(stmt), pc::Int) mod = moduleof(frame) @@ -52,9 +52,10 @@ function signature(interp::Interpreter, frame::Frame, @nospecialize(stmt), pc::I stmt = pc_expr(frame, pc) end isa(stmt, Expr) || return nothing, pc + mt = extract_method_table(frame, stmt) sigsv = lookup(interp, frame, stmt.args[2])::SimpleVector sigt = signature(sigsv) - return sigt, lastpc + return MethodInfoKey(mt, sigt), lastpc end signature(interp::Interpreter, frame::Frame, pc::Int) = signature(interp, frame, pc_expr(frame, pc), pc) signature(frame::Frame, pc::Int) = signature(RecursiveInterpreter(), frame, pc) @@ -187,7 +188,9 @@ function identify_framemethod_calls(frame::Frame) end msrc = stmt.args[3] if msrc isa CodeInfo - key = key::Union{GlobalRef,Bool,Nothing} + # XXX: Properly support interpolated `Core.MethodTable`. This will require using + # `stmt.args[2]` instead of `stmt.args[1]` to identify the parent function. + isa(key, Union{GlobalRef,Bool,Nothing}) || continue for (j, mstmt) in enumerate(msrc.code) isa(mstmt, Expr) || continue jj = j @@ -444,9 +447,9 @@ function get_running_name(interp::Interpreter, frame::Frame, pc::Int, name::Glob pctop -= 1 stmt = pc_expr(frame, pctop) end # end fix - sigtparent, lastpcparent = signature(interp, frame, pctop) + (mt, sigtparent), lastpcparent = signature(interp, frame, pctop) sigtparent === nothing && return name, pc, lastpcparent - methparent = whichtt(sigtparent) + methparent = whichtt(sigtparent, mt) methparent === nothing && return name, pc, lastpcparent # caller isn't defined, no correction is needed if isgen cname = GlobalRef(moduleof(frame), nameof(methparent.generator.gen)) @@ -515,8 +518,8 @@ end """ ret = methoddef!([interp::Interpreter=RecursiveInterpreter()], signatures, frame; define=true) -Compute the signature of a method definition. `frame.pc` should point to a -`:method` expression. Upon exit, the new signature will be added to `signatures`. +Compute the method table/signature pair of a method definition. `frame.pc` should point to a +`:method` expression. Upon exit, the new method table/signature pair will be added to `signatures`. There are several possible return values: @@ -535,27 +538,27 @@ occurs for "empty method" expressions, e.g., `:(function foo end)`. `pc` will be By default the method will be defined (evaluated). You can prevent this by setting `define=false`. This is recommended if you are simply extracting signatures from code that has already been evaluated. """ -function methoddef!(interp::Interpreter, signatures, frame::Frame, @nospecialize(stmt), pc::Int; define::Bool=true) +function methoddef!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame, @nospecialize(stmt), pc::Int; define::Bool=true) framecode, pcin = frame.framecode, pc if ismethod3(stmt) pc3 = pc arg1 = stmt.args[1] - sigt, pc = signature(interp, frame, stmt, pc) - meth = whichtt(sigt) + (mt, sigt), pc = signature(interp, frame, stmt, pc) + meth = whichtt(sigt, mt) if isa(meth, Method) && (meth.sig <: sigt && sigt <: meth.sig) pc = define ? step_expr!(interp, frame, stmt, true) : next_or_nothing!(interp, frame) elseif define pc = step_expr!(interp, frame, stmt, true) - meth = whichtt(sigt) + meth = whichtt(sigt, mt) end if isa(meth, Method) && (meth.sig <: sigt && sigt <: meth.sig) - push!(signatures, meth.sig) + push!(signatures, mt => meth.sig) else - if arg1 === false || arg1 === nothing + if arg1 === false || arg1 === nothing || isa(mt, MethodTable) # If it's anonymous and not defined, define it pc = step_expr!(interp, frame, stmt, true) - meth = whichtt(sigt) - isa(meth, Method) && push!(signatures, meth.sig) + meth = whichtt(sigt, mt) + isa(meth, Method) && push!(signatures, mt => meth.sig) return pc, pc3 else # guard against busted lookup, e.g., https://github.com/JuliaLang/julia/issues/31112 @@ -596,7 +599,7 @@ function methoddef!(interp::Interpreter, signatures, frame::Frame, @nospecialize end found || return nothing while true # methods containing inner methods may need multiple trips through this loop - sigt, pc = signature(interp, frame, stmt, pc) + (mt, sigt), pc = signature(interp, frame, stmt, pc) stmt = pc_expr(frame, pc) while !isexpr(stmt, :method, 3) pc = next_or_nothing(interp, frame, pc) # this should not check define, we've probably already done this once @@ -611,15 +614,15 @@ function methoddef!(interp::Interpreter, signatures, frame::Frame, @nospecialize # signature of the active method. So let's get the active signature. frame.pc = pc pc = define ? step_expr!(interp, frame, stmt, true) : next_or_nothing!(interp, frame) - meth = whichtt(sigt) - isa(meth, Method) && push!(signatures, meth.sig) # inner methods are not visible + meth = whichtt(sigt, mt) + isa(meth, Method) && push!(signatures, mt => meth.sig) # inner methods are not visible name === name3 && return pc, pc3 # if this was an inner method we should keep going stmt = pc_expr(frame, pc) # there *should* be more statements in this frame end end -methoddef!(interp::Interpreter, signatures, frame::Frame, pc::Int; define::Bool=true) = +methoddef!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame, pc::Int; define::Bool=true) = methoddef!(interp, signatures, frame, pc_expr(frame, pc), pc; define) -function methoddef!(interp::Interpreter, signatures, frame::Frame; define::Bool=true) +function methoddef!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame; define::Bool=true) pc = frame.pc stmt = pc_expr(frame, pc) if !ismethod(stmt) @@ -628,27 +631,27 @@ function methoddef!(interp::Interpreter, signatures, frame::Frame; define::Bool= pc === nothing && error("pc at end of frame without finding a method") methoddef!(interp, signatures, frame, pc; define) end -methoddef!(signatures, frame::Frame, pc::Int; define::Bool=true) = +methoddef!(signatures::Vector{MethodInfoKey}, frame::Frame, pc::Int; define::Bool=true) = methoddef!(RecursiveInterpreter(), signatures, frame, pc_expr(frame, pc), pc; define) -methoddef!(signatures, frame::Frame; define::Bool=true) = +methoddef!(signatures::Vector{MethodInfoKey}, frame::Frame; define::Bool=true) = methoddef!(RecursiveInterpreter(), signatures, frame; define) -function methoddefs!(interp::Interpreter, signatures, frame::Frame, pc::Int; define::Bool=true) +function methoddefs!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame, pc::Int; define::Bool=true) ret = methoddef!(interp, signatures, frame, pc; define) pc = ret === nothing ? ret : ret[1] return _methoddefs!(interp, signatures, frame, pc; define) end -function methoddefs!(interp::Interpreter, signatures, frame::Frame; define::Bool=true) +function methoddefs!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame; define::Bool=true) ret = methoddef!(interp, signatures, frame; define) pc = ret === nothing ? ret : ret[1] return _methoddefs!(interp, signatures, frame, pc; define) end -methoddefs!(signatures, frame::Frame, pc::Int; define::Bool=true) = +methoddefs!(signatures::Vector{MethodInfoKey}, frame::Frame, pc::Int; define::Bool=true) = methoddefs!(RecursiveInterpreter(), signatures, frame, pc; define) -methoddefs!(signatures, frame::Frame; define::Bool=true) = +methoddefs!(signatures::Vector{MethodInfoKey}, frame::Frame; define::Bool=true) = methoddefs!(RecursiveInterpreter(), signatures, frame; define) -function _methoddefs!(interp::Interpreter, signatures, frame::Frame, pc::Int; define::Bool=define) +function _methoddefs!(interp::Interpreter, signatures::Vector{MethodInfoKey}, frame::Frame, pc::Int; define::Bool=define) while pc !== nothing stmt = pc_expr(frame, pc) if !ismethod(stmt) diff --git a/test/codeedges.jl b/test/codeedges.jl index 412620a..f1a9437 100644 --- a/test/codeedges.jl +++ b/test/codeedges.jl @@ -348,7 +348,7 @@ module ModSelective end edges = CodeEdges(ModEval, src) lr = lines_required(GlobalRef(ModEval, :revise538), src, edges) selective_eval_fromstart!(Frame(ModEval, src), lr, #=istoplevel=#true) - @test isdefined(ModEval, :revise538) && length(methods(ModEval.revise538, (Float32,))) == 1 + @test isdefined(ModEval, :revise538) && isempty(methods(ModEval.revise538)) # function is defined, method is not # https://github.com/timholy/Revise.jl/issues/599 thk = Meta.lower(Main, quote diff --git a/test/signatures.jl b/test/signatures.jl index 259c0ae..05ea97c 100644 --- a/test/signatures.jl +++ b/test/signatures.jl @@ -2,6 +2,7 @@ module Signatures using LoweredCodeUtils using InteractiveUtils +using CodeTracking: MethodInfoKey using JuliaInterpreter using Core: CodeInfo using Base.Meta: isexpr @@ -33,7 +34,7 @@ bodymethtest4(x, y=1) = 4 bodymethtest5(x, y=Dict(1=>2)) = 5 @testset "Signatures" begin - signatures = Set{Any}() + signatures = MethodInfoKey[] newcode = CodeInfo[] for ex in (:(f(x::Int8; y=0) = y), :(f(x::Int16; y::Int=0) = 2), @@ -96,7 +97,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 # Manually add the signature for the Caller constructor, since that was defined # outside of manual lowering - push!(signatures, Tuple{Type{Lowering.Caller}}) + push!(signatures, nothing => Tuple{Type{Lowering.Caller}}) nms = names(Lowering; all=true) modeval, modinclude = getfield(Lowering, :eval), getfield(Lowering, :include) @@ -106,7 +107,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 isa(f, Base.Callable) || continue (f === modeval || f === modinclude) && continue for m in methods(f) - if m.sig ∉ signatures + if (nothing => m.sig) ∉ signatures push!(failed, m.sig) end end @@ -138,7 +139,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 @test g(3) == 6 # Don't be deceived by inner methods - signatures = [] + signatures = MethodInfoKey[] ex = quote function fouter(x) finner(::Float16) = 2x @@ -150,7 +151,8 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 rename_framemethods!(frame) methoddefs!(signatures, frame; define=false) @test length(signatures) == 1 - @test LoweredCodeUtils.whichtt(signatures[1]) == first(methods(Lowering.fouter)) + mt, sig = first(signatures) + @test LoweredCodeUtils.whichtt(sig, mt) == first(methods(Lowering.fouter)) # Check output of methoddef! frame = Frame(Lowering, :(function nomethod end)) @@ -166,10 +168,11 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 ex = :(max_values(T::Union{map(X -> Type{X}, Base.BitIntegerSmall_types)...}) = 1 << (8*sizeof(T))) # base/abstractset.jl frame = Frame(Base, ex) rename_framemethods!(frame) - signatures = Set{Any}() + signatures = MethodInfoKey[] methoddef!(signatures, frame; define=false) @test length(signatures) == 1 - @test first(signatures) == which(Base.max_values, Tuple{Type{Int16}}).sig + mt, sig = first(signatures) + @test sig == which(Base.max_values, Tuple{Type{Int16}}).sig # define ex = :(fdefine(x) = 1) @@ -290,7 +293,8 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 JuliaInterpreter.next_until!(LoweredCodeUtils.ismethod3, frame, true) empty!(signatures) methoddefs!(signatures, frame; define=true) - @test first(signatures).parameters[end] == Int + mt, sig = first(signatures) + @test sig.parameters[end] == Int # Multiple keyword arg methods per frame # (Revise issue #363) @@ -309,7 +313,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 @test kw2sig ∉ signatures pc = methoddefs!(signatures, frame; define=false) @test pc === nothing - @test kw2sig ∈ signatures + @test (nothing => kw2sig) ∈ signatures # Module-scoping ex = :(Base.@irrational π 3.14159265358979323846 pi) @@ -335,7 +339,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 rename_framemethods!(frame) empty!(signatures) methoddefs!(signatures, frame; define=false) - @test Tuple{typeof(Lowering.CustomMS)} ∈ signatures + @test (nothing => Tuple{typeof(Lowering.CustomMS)}) ∈ signatures # https://github.com/timholy/Revise.jl/issues/398 ex = quote @@ -369,7 +373,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 frame = Frame(Lowering422, ex) rename_framemethods!(frame) pc = methoddefs!(signatures, frame; define=false) - @test typeof(Lowering422.fneg) ∈ Set(Base.unwrap_unionall(sig).parameters[1] for sig in signatures) + @test typeof(Lowering422.fneg) ∈ Set(Base.unwrap_unionall(sig).parameters[1] for (mt, sig) in signatures) # Scoped names (https://github.com/timholy/Revise.jl/issues/568) ex = :(f568() = -1) @@ -384,7 +388,7 @@ bodymethtest5(x, y=Dict(1=>2)) = 5 pc = JuliaInterpreter.step_expr!(frame, true) end pc = methoddef!(signatures, frame, pc; define=true) - @test Tuple{typeof(Lowering.f568)} ∈ signatures + @test (nothing => Tuple{typeof(Lowering.f568)}) ∈ signatures @test Lowering.f568() == -2 # Undefined names @@ -500,4 +504,38 @@ end end +module ExternalMT + Base.Experimental.@MethodTable method_table + macro overlay(ex) esc(:(Base.Experimental.@overlay $method_table $ex)) end +end + +@testset "Support for external method tables" begin + signatures = MethodInfoKey[] + + ex = :(foo(x) = "foo") + Core.eval(ExternalMT, ex) + frame = Frame(ExternalMT, ex) + pc = methoddefs!(signatures, frame; define = false) + @test length(signatures) == 1 + (mt, sig) = pop!(signatures) + @test (mt, sig) === (nothing, Tuple{typeof(ExternalMT.foo), Any}) + + ex = :(Base.Experimental.@overlay method_table foo(x) = "overlayed foo") + Core.eval(ExternalMT, ex) + frame = Frame(ExternalMT, ex) + pc = methoddefs!(signatures, frame; define = false) + @test length(signatures) == 1 + (mt, sig) = pop!(signatures) + @test (mt, sig) === (ExternalMT.method_table, Tuple{typeof(ExternalMT.foo), Any}) + + ex = :(@overlay foo(x::Int64) = "overlayed foo, second edition") + Core.eval(ExternalMT, ex) + frame = Frame(ExternalMT, ex) + pc = methoddefs!(signatures, frame; define = false) + @test length(signatures) == 1 + (mt, sig) = pop!(signatures) + @test (mt, sig) === (ExternalMT.method_table, Tuple{typeof(ExternalMT.foo), Int64}) + LoweredCodeUtils.identify_framemethod_calls(frame) # make sure this does not throw +end + end # module signatures