Skip to content

nondeterminism in invalidation between recompilations of Julia #57716

@nsajko

Description

@nsajko

Rebuilding Julia may lead to different behavior on method invalidation. What I did:

  1. Initial build of master
    1. Build master (d369c10) using make MARCH=native MCPU=native MTUNE=native
    2. Measure invalidations on loading the package TypeDomainNaturalNumbers v6.1.0 using the script provided below: there are 966 invalidations. Data on each invalidation in JSON format: master__d369c10c6f__1.json
  2. Build some other change on top of d369c10, again by running make MARCH=native MCPU=native MTUNE=native
  3. Build d369c10 with make MARCH=native MCPU=native MTUNE=native again. Now there are only 965 invalidations on loading the same package.
    • Data on each invalidation in JSON format: master__d369c10c6f__2.json
    • The specific invalidation that's now gone is:
      {
          "method": ">>(m::TypeDomainNaturalNumbers.LazyMinus.TypeDomainInteger, n::Int64) @ TypeDomainNaturalNumbers.BaseOverloads ~/src/gitlab.com/nsajko/TypeDomainNaturalNumbers/src/TypeDomainNaturalNumbers.jl:1518",
          "reason": "inserting",
          "mt_backedges": [
              {
                  "type": "Tuple{typeof(>>), Any, Int64}",
                  "tree": {
                      "method_instance": {
                          "method": "var\"#sizehint!#163\"(first::Bool, shrink::Bool, ::typeof(sizehint!), s::BitSet, n::Integer) @ Base bitset.jl:58",
                          "method_instance": "MethodInstance for Base.var\"#sizehint!#163\"(::Bool, ::Bool, ::typeof(sizehint!), ::BitSet, ::Integer)"
                      },
                      "children": [
                      ]
                  }
              }
          ],
          "backedges": [
          ],
          "mt_cache": [
          ],
          "mt_disable": [
          ]
      }

The script I use to measure invalidations:

  • records the invalidation count, as length(uinvalidated(raw_invalidations)), where uinvalidated is from the SnoopCompile package, and raw_invalidations is the output of @snoop_invalidations using TypeDomainNaturalNumbers
  • records a straight translation of the output of invalidation_trees, from the SnoopCompile package, as JSON
Script source
module A
    using SnoopCompileCore: @snoop_invalidations
    raw_invalidations = @snoop_invalidations begin
        using TypeDomainNaturalNumbers
    end
end

module B
    export comprehensive_to_named_tuple_trees
    function comprehensive_to_named_tuple_tree_method_instance(mi)
        method = string(mi.def)
        method_instance = string(mi)
        (; method, method_instance)
    end
    function comprehensive_to_named_tuple_tree_method_instances(method_instances)
        r = []
        for mi  method_instances
            t = comprehensive_to_named_tuple_tree_method_instance(mi)
            push!(r, t)
        end
        sort!(r)
    end
    function comprehensive_to_named_tuple_tree_node(node)
        method_instance = comprehensive_to_named_tuple_tree_method_instance(node.mi)
        children = comprehensive_to_named_tuple_tree_nodes(node.children)
        (; method_instance, children)
    end
    function comprehensive_to_named_tuple_tree_nodes(nodes)
        r = []
        for node  nodes
            t = comprehensive_to_named_tuple_tree_node(node)
            push!(r, t)
        end
        sort!(r)
    end
    function comprehensive_to_named_tuple_tree_mt_backedges(invalidations)
        r = []
        for (type_raw, mi_or_node)  invalidations
            type = string(type_raw)
            tree = if mi_or_node isa Core.MethodInstance
                comprehensive_to_named_tuple_tree_method_instance(mi_or_node)
            else
                comprehensive_to_named_tuple_tree_node(mi_or_node)
            end
            t = (; type, tree)
            push!(r, t)
        end
        sort!(r)
    end
    function comprehensive_to_named_tuple_tree(tree)
        method = string(tree.method)
        reason = string(tree.reason)
        mt_backedges = comprehensive_to_named_tuple_tree_mt_backedges(tree.mt_backedges)
        backedges = comprehensive_to_named_tuple_tree_nodes(tree.backedges)
        mt_cache = comprehensive_to_named_tuple_tree_method_instances(tree.mt_cache)
        mt_disable = comprehensive_to_named_tuple_tree_method_instances(tree.mt_disable)
        (; method, reason, mt_backedges, backedges, mt_cache, mt_disable)
    end
    function comprehensive_to_named_tuple_trees(trees)
        r = []
        for tree  trees
            t = comprehensive_to_named_tuple_tree(tree)
            push!(r, t)
        end
        sort!(r)
    end
end

module C
    using SnoopCompile: uinvalidated, invalidation_trees
    using ..B
    function comprehensive_to_named_tuple(raw_invalidations)
        invalidation_count = length(uinvalidated(raw_invalidations))
        trees = comprehensive_to_named_tuple_trees(invalidation_trees(raw_invalidations))
        (; invalidation_count, trees)
    end
end

using JSON3: pretty
pretty(C.comprehensive_to_named_tuple(A.raw_invalidations))

Metadata

Metadata

Assignees

No one assigned

    Type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions