more effectively record precompile signatures from package precompilation (#37710)

* more effectively record precompile signatures from package precompilation

Author: KristofferC

base/loading.jl

@@ -1152,6 +1152,7 @@ end
 @assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),Nothing))
 @assert precompile(include_package_for_output, (PkgId,String,Vector{String},Vector{String},Vector{String},typeof(_concrete_dependencies),String))
 
+const PRECOMPILE_TRACE_COMPILE = Ref{String}()
 function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), show_errors::Bool = true)
     rm(output, force=true)   # Remove file if it exists
     depot_path = map(abspath, DEPOT_PATH)
@@ -1174,10 +1175,12 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d
     end
     deps = repr(eltype(concrete_deps)) * "[" * join(deps_strs, ",") * "]"
 
+    trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : ``
     io = open(pipeline(`$(julia_cmd()::Cmd) -O0
                        --output-ji $output --output-incremental=yes
                        --startup-file=no --history-file=no --warn-overwrite=yes
                        --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no")
+                       $trace
                        --eval 'eval(Meta.parse(read(stdin,String)))'`, stderr=show_errors ? stderr : devnull),
               "w", stdout)
     # write data over stdin to avoid the (unlikely) case of exceeding max command line size

contrib/generate_precompile.jl

@@ -122,21 +122,31 @@ end
 function generate_precompile_statements()
     start_time = time_ns()
     debug_output = devnull # or stdout
+    sysimg = Base.unsafe_string(Base.JLOptions().image_file)
 
     # Precompile a package
+    global hardcoded_precompile_statements
+
     mktempdir() do prec_path
-        push!(DEPOT_PATH, prec_path)
-        push!(LOAD_PATH, prec_path)
         pkgname = "__PackagePrecompilationStatementModule"
         mkpath(joinpath(prec_path, pkgname, "src"))
-        write(joinpath(prec_path, pkgname, "src", "$pkgname.jl"),
+        path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
+        write(path,
               """
               module $pkgname
               end
               """)
-        @eval using __PackagePrecompilationStatementModule
-        empty!(LOAD_PATH)
-        empty!(DEPOT_PATH)
+        tmp = tempname()
+        # Running compilecache on buildbots fails with
+        # `More than one command line CPU targets specified without a `--output-` flag specified`
+        # so start a new process without a CPU target specified
+        s = """
+            push!(DEPOT_PATH, $(repr(prec_path)));
+            Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp));
+            Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
+            """
+        run(`$(julia_exepath()) -O0 --sysimage $sysimg -Cnative -e $s`)
+        hardcoded_precompile_statements *= "\n" * read(tmp, String)
     end
 
     mktemp() do precompile_file, precompile_file_h
@@ -154,7 +164,6 @@ function generate_precompile_statements()
                     "JULIA_PROJECT" => nothing, # remove from environment
                     "JULIA_LOAD_PATH" => Sys.iswindows() ? "@;@stdlib" : "@:@stdlib",
                     "TERM" => "") do
-            sysimg = Base.unsafe_string(Base.JLOptions().image_file)
             run(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
                    --cpu-target=native --startup-file=no --color=yes
                    -e 'import REPL; REPL.Terminals.is_precompiling[] = true'

test/precompile.jl

@@ -293,10 +293,6 @@ try
                  :Profile, :Random, :Serialization, :SharedArrays, :SparseArrays, :SuiteSparse, :Test,
                  :Unicode, :REPL, :InteractiveUtils, :Pkg, :LibGit2, :SHA, :UUIDs, :Sockets,
                  :Statistics, :TOML, :MozillaCACerts_jll, :LibCURL_jll, :LibCURL, :Downloads,]),
-                # Plus precompilation module generated at build time
-                let id = Base.PkgId("__PackagePrecompilationStatementModule")
-                    Dict(id => Base.module_build_id(Base.root_module(id)))
-                end
            )
         @test discard_module.(deps) == deps1
         modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile; srcfiles_only=true)