Iterate over readmeta results (#1308)

eschnett · giordano · commit 8d917f96746f · 2024-01-14T19:11:25.000Z
* Iterate over `readmeta` results

* Correct get_soname

* Require ObjectFile 0.4

* Update Manifest.toml
diff --git a/Manifest.toml b/Manifest.toml
@@ -2,7 +2,7 @@
 
 julia_version = "1.7.2"
 manifest_format = "2.0"
-project_hash = "1e87134dc5853b26c2f65af5a46eb0a2d987da86"
+project_hash = "37b1c77f30c4c87e9820261f70d049fd61c325f2"
 
 [[deps.ArgParse]]
 deps = ["Logging", "TextWrap"]
@@ -38,7 +38,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[deps.BinaryBuilderBase]]
 deps = ["Bzip2_jll", "CodecZlib", "Downloads", "Gzip_jll", "HistoricalStdlibVersions", "InteractiveUtils", "JLLWrappers", "JSON", "LibGit2", "LibGit2_jll", "Libdl", "Logging", "OrderedCollections", "OutputCollectors", "Pkg", "Printf", "ProgressMeter", "REPL", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "Tar_jll", "UUIDs", "XZ_jll", "Zstd_jll", "p7zip_jll", "pigz_jll", "unzip_jll"]
-git-tree-sha1 = "b6e70b712721dea3f586d97aa05e7b5eba96309c"
+git-tree-sha1 = "6f8c4fb870b394c018455d05c2f6f9097f80175b"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
@@ -62,10 +62,10 @@ uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.3"
 
 [[deps.Compat]]
-deps = ["Dates", "LinearAlgebra", "UUIDs"]
-git-tree-sha1 = "886826d76ea9e72b35fcd000e535588f7b60f21d"
+deps = ["Dates", "LinearAlgebra", "TOML", "UUIDs"]
+git-tree-sha1 = "75bd5b6fc5089df449b5d35fa501c846c9b6549b"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.10.1"
+version = "4.12.0"
 
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -96,14 +96,14 @@ deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[deps.Downloads]]
-deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
 [[deps.ExceptionUnwrapping]]
 deps = ["Test"]
-git-tree-sha1 = "e90caa41f5a86296e014e148ee061bd6c3edec96"
+git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a"
 uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4"
-version = "0.1.9"
+version = "0.1.10"
 
 [[deps.ExprTools]]
 git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
@@ -112,9 +112,9 @@ version = "0.1.10"
 
 [[deps.FileIO]]
 deps = ["Pkg", "Requires", "UUIDs"]
-git-tree-sha1 = "299dc33549f68299137e51e6d49a13b5b1da9673"
+git-tree-sha1 = "c5c28c245101bd59154f649e19b038d15901b5dc"
 uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
-version = "1.16.1"
+version = "1.16.2"
 
 [[deps.FileWatching]]
 uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
@@ -171,9 +171,9 @@ version = "1.0.0"
 
 [[deps.JLD2]]
 deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "Pkg", "PrecompileTools", "Printf", "Reexport", "Requires", "TranscodingStreams", "UUIDs"]
-git-tree-sha1 = "c2d0f45afcb5f6209155670bffd100c3b4937ea3"
+git-tree-sha1 = "853b9e0b876e8c7093a824c4f83c619d23525e64"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.40"
+version = "0.4.43"
 
 [[deps.JLLWrappers]]
 deps = ["Artifacts", "Preferences"]
@@ -288,9 +288,9 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 
 [[deps.ObjectFile]]
 deps = ["Reexport", "StructIO"]
-git-tree-sha1 = "55ce61d43409b1fb0279d1781bf3b0f22c83ab3b"
+git-tree-sha1 = "195e0a19842f678dd3473ceafbe9d82dfacc583c"
 uuid = "d8793406-e978-5875-9003-1fc021f44a92"
-version = "0.3.7"
+version = "0.4.1"
 
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
@@ -555,10 +555,10 @@ deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
 
 [[deps.pigz_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
-git-tree-sha1 = "3c0c0b0c133b6ab53e1af05dc526091ce8781f16"
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"]
+git-tree-sha1 = "cb56f0446caa6138418ee28132024e2c3911d19b"
 uuid = "1bc43ea1-30af-5bc8-a9d4-c018457e6e3e"
-version = "2.7.0+0"
+version = "2.8.0+0"
 
 [[deps.unzip_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl"]
diff --git a/Project.toml b/Project.toml
@@ -41,7 +41,7 @@ JLD2 = "0.1.6, 0.2, 0.3, 0.4"
 JLLWrappers = "1.2.0"
 JSON = "0.21"
 LoggingExtras = "0.4, 1"
-ObjectFile = "0.3.6, 0.4"
+ObjectFile = "0.4"
 OutputCollectors = "0.1"
 PkgLicenses = "0.2"
 Registrator = "1.1"
diff --git a/src/Auditor.jl b/src/Auditor.jl
@@ -81,38 +81,40 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
 
         # Peel this binary file open like a delicious tangerine
         try
-            readmeta(f) do oh
-                if !is_for_platform(oh, platform)
-                    if verbose
-                        @warn("Skipping binary analysis of $(relpath(f, prefix.path)) (incorrect platform)")
-                    end
-                else
-                    # Check that the ISA isn't too high
-                    all_ok &= check_isa(oh, platform, prefix; verbose, silent)
-                    # Check that the OS ABI is set correctly (often indicates the wrong linker was used)
-                    all_ok &= check_os_abi(oh, platform; verbose)
-                    # Make sure all binary files are executables, if libraries aren't
-                    # executables Julia may not be able to dlopen them:
-                    # https://github.com/JuliaLang/julia/issues/38993.  In principle this
-                    # should be done when autofix=true, but we have to run this fix on MKL
-                    # for Windows, for which however we have to set autofix=false:
-                    # https://github.com/JuliaPackaging/Yggdrasil/pull/922.
-                    all_ok &= ensure_executability(oh; verbose, silent)
-
-                    # If this is a dynamic object, do the dynamic checks
-                    if isdynamic(oh)
-                        # Check that the libgfortran version matches
-                        all_ok &= check_libgfortran_version(oh, platform; verbose, has_csl)
-                        # Check whether the library depends on any of the most common
-                        # libraries provided by `CompilerSupportLibraries_jll`.
-                        all_ok &= check_csl_libs(oh, platform; verbose, has_csl)
-                        # Check that the libstdcxx string ABI matches
-                        all_ok &= check_cxxstring_abi(oh, platform; verbose)
-                        # Check that this binary file's dynamic linkage works properly.  Note to always
-                        # DO THIS ONE LAST as it can actually mutate the file, which causes the previous
-                        # checks to freak out a little bit.
-                        all_ok &= check_dynamic_linkage(oh, prefix, bin_files;
-                                                        platform, silent, verbose, autofix, src_name)
+            readmeta(f) do ohs
+                foreach(ohs) do oh
+                    if !is_for_platform(oh, platform)
+                        if verbose
+                            @warn("Skipping binary analysis of $(relpath(f, prefix.path)) (incorrect platform)")
+                        end
+                    else
+                        # Check that the ISA isn't too high
+                        all_ok &= check_isa(oh, platform, prefix; verbose, silent)
+                        # Check that the OS ABI is set correctly (often indicates the wrong linker was used)
+                        all_ok &= check_os_abi(oh, platform; verbose)
+                        # Make sure all binary files are executables, if libraries aren't
+                        # executables Julia may not be able to dlopen them:
+                        # https://github.com/JuliaLang/julia/issues/38993.  In principle this
+                        # should be done when autofix=true, but we have to run this fix on MKL
+                        # for Windows, for which however we have to set autofix=false:
+                        # https://github.com/JuliaPackaging/Yggdrasil/pull/922.
+                        all_ok &= ensure_executability(oh; verbose, silent)
+                    
+                        # If this is a dynamic object, do the dynamic checks
+                        if isdynamic(oh)
+                            # Check that the libgfortran version matches
+                            all_ok &= check_libgfortran_version(oh, platform; verbose, has_csl)
+                            # Check whether the library depends on any of the most common
+                            # libraries provided by `CompilerSupportLibraries_jll`.
+                            all_ok &= check_csl_libs(oh, platform; verbose, has_csl)
+                            # Check that the libstdcxx string ABI matches
+                            all_ok &= check_cxxstring_abi(oh, platform; verbose)
+                            # Check that this binary file's dynamic linkage works properly.  Note to always
+                            # DO THIS ONE LAST as it can actually mutate the file, which causes the previous
+                            # checks to freak out a little bit.
+                            all_ok &= check_dynamic_linkage(oh, prefix, bin_files;
+                                                            platform, silent, verbose, autofix, src_name)
+                        end
                     end
                 end
             end
diff --git a/src/auditor/compiler_abi.jl b/src/auditor/compiler_abi.jl
@@ -120,8 +120,8 @@ function detect_libstdcxx_version(oh::ObjectHandle, platform::AbstractPlatform)
 
     # Extract all pieces of `.gnu.version_d` from libstdc++.so, find the `GLIBCXX_*`
     # symbols, and use the maximum version of that to find the GLIBCXX ABI version number
-    version_symbols = readmeta(first(libstdcxx_libs)) do oh
-        unique(vcat((x -> x.names).(ELFVersionData(oh))...))
+    version_symbols = readmeta(first(libstdcxx_libs)) do ohs
+        unique(vcat((x -> x.names).(vcat(ELFVersionData.(ohs)...))...))
     end
     version_symbols = filter(x -> startswith(x, "GLIBCXX_"), version_symbols)
     if isempty(version_symbols)
diff --git a/src/auditor/dynamic_linkage.jl b/src/auditor/dynamic_linkage.jl
@@ -68,14 +68,14 @@ function platform_for_object(oh::ObjectHandle)
 end
 
 function _rpaths(file::AbstractString)
-    readmeta(file) do oh
-        rpaths(RPath(oh))
+    readmeta(file) do ohs
+        vcat(rpaths.(RPath.(ohs))...)
     end
 end
 
 function _canonical_rpaths(file::AbstractString)
-    readmeta(file) do oh
-        canonical_rpaths(RPath(oh))
+    readmeta(file) do ohs
+        vcat(canonical_rpaths.(RPath.(ohs))...)
     end
 end
 
diff --git a/src/auditor/soname_matching.jl b/src/auditor/soname_matching.jl
@@ -4,7 +4,7 @@ get_soname(oh::ObjectHandle) = nothing
 # Auto-open a path into an ObjectHandle
 function get_soname(path::AbstractString)
     try
-        readmeta(get_soname, path)
+        only(readmeta(ns -> get_soname.(ns), path))
     catch e
         @warn "Could not probe $(path) for an SONAME!" exception=(e, catch_backtrace())
         return nothing
diff --git a/src/wizard/interactive_build.jl b/src/wizard/interactive_build.jl
@@ -34,15 +34,16 @@ function step4(state::WizardState, ur::Runner, platform::AbstractPlatform,
 
     # Check if we can load them as an object file
     files = filter(files) do f
-        readmeta(f) do oh
-            return Auditor.is_for_platform(oh, platform)
+        readmeta(f) do ohs
+            return any(Auditor.is_for_platform(oh, platform) for oh in ohs)
         end
     end
 
     # Strip out the prefix from filenames
     state.files = map(file->replace(file, "$(destdir_path)/" => ""), files)
     state.file_kinds = map(files) do f
-        readmeta(f) do oh
+        readmeta(f) do ohs
+            oh = first(ohs)
             if isexecutable(oh)
                 return :executable
             elseif islibrary(oh)
diff --git a/src/wizard/utils.jl b/src/wizard/utils.jl
@@ -85,7 +85,7 @@ from `ObjectFile`.
 function filter_object_files(files)
     return filter(files) do f
         try
-            readmeta(f) do oh
+            readmeta(f) do ohs
                 return true
             end
         catch e
@@ -121,8 +121,8 @@ function match_files(state::WizardState, prefix::Prefix,
     prefix_files = filter_object_files(prefix_files)
     # Check if we can load them as an object file
     prefix_files = filter(prefix_files) do f
-        readmeta(f) do oh
-            if !Auditor.is_for_platform(oh, platform)
+        readmeta(f) do ohs
+            if !any(Auditor.is_for_platform(oh, platform) for oh in ohs)
                 if !silent
                     @warn("Skipping binary `$f` with incorrect platform")
                 end
diff --git a/test/auditing.jl b/test/auditing.jl
@@ -86,9 +86,11 @@ end
             prefix = Prefix(testdir)
 
             # Run ISA test
-            readmeta(locate(product, prefix)) do oh
-                detected_isa = Auditor.analyze_instruction_set(oh, platform; verbose=true)
-                @test detected_isa == "avx512"
+            readmeta(locate(product, prefix)) do ohs
+                foreach(ohs) do oh
+                    detected_isa = Auditor.analyze_instruction_set(oh, platform; verbose=true)
+                    @test detected_isa == "avx512"
+                end
             end
         end
     end
@@ -133,9 +135,11 @@ end
             prefix = Prefix(testdir)
 
             # Run ISA test
-            readmeta(locate(product, prefix)) do oh
-                detected_march = Auditor.analyze_instruction_set(oh, platform; verbose=true)
-                @test detected_march == "avx"
+            readmeta(locate(product, prefix)) do ohs
+                foreach(ohs) do oh
+                    detected_march = Auditor.analyze_instruction_set(oh, platform; verbose=true)
+                    @test detected_march == "avx"
+                end
             end
         end
     end
@@ -182,14 +186,16 @@ end
             prefix = Prefix(testdir)
 
             # Run ISA test
-            readmeta(locate(product, prefix)) do oh
-                detected_march = Auditor.analyze_instruction_set(oh, platform; verbose=true)
-                if march == "avx2"
-                    # Detecting the ISA isn't 100% reliable and it's even less
-                    # accurate when looking for AVX2 features
-                    @test_broken march == detected_march
-                else
-                    @test march == detected_march
+            readmeta(locate(product, prefix)) do ohs
+                foreach(ohs) do oh
+                    detected_march = Auditor.analyze_instruction_set(oh, platform; verbose=true)
+                    if march == "avx2"
+                        # Detecting the ISA isn't 100% reliable and it's even less
+                        # accurate when looking for AVX2 features
+                        @test_broken march == detected_march
+                    else
+                        @test march == detected_march
+                    end
                 end
             end
         end
@@ -243,9 +249,11 @@ end
                 prefix = Prefix(testdir)
 
                 # Ensure that the library detects as the correct cxxstring_abi:
-                readmeta(locate(libcxxstringabi_test, prefix)) do oh
-                    detected_cxxstring_abi = Auditor.detect_cxxstring_abi(oh, platform)
-                    @test detected_cxxstring_abi == cxxstring_abi(platform)
+                readmeta(locate(libcxxstringabi_test, prefix)) do ohs
+                    foreach(ohs) do oh
+                        detected_cxxstring_abi = Auditor.detect_cxxstring_abi(oh, platform)
+                        @test detected_cxxstring_abi == cxxstring_abi(platform)
+                    end
                 end
 
                 # Explicitly test cxx string abi mismatches
@@ -404,11 +412,13 @@ end
         prefix = Prefix(testdir)
 
         # Helper to extract the dylib id of a path
-        function get_dylib_id(path)
-            return readmeta(path) do oh
-                dylib_id_lcs = [lc for lc in MachOLoadCmds(oh) if isa(lc, MachOIdDylibCmd)]
-                @test !isempty(dylib_id_lcs)
-                return dylib_name(first(dylib_id_lcs))
+        function get_dylib_ids(path)
+            return readmeta(path) do ohs
+                map(ohs) do oh
+                     dylib_id_lcs = [lc for lc in MachOLoadCmds(oh) if isa(lc, MachOIdDylibCmd)]
+                     @test !isempty(dylib_id_lcs)
+                     return dylib_name(first(dylib_id_lcs))
+                end
             end
         end
 
@@ -419,7 +429,7 @@ end
         right_id_path = locate(right_id, prefix; platform=platform)
         for p in (no_id_path, abs_id_path, right_id_path)
             @test any(startswith.(p, libdirs(prefix)))
-            @test get_dylib_id(p) == "@rpath/$(basename(p))"
+            @test all(get_dylib_ids(p) .== "@rpath/$(basename(p))")
         end
 
         # Only if it already has an `@rpath/`-ified ID, it doesn't get touched.
@@ -534,10 +544,12 @@ end
         # audit should warn us.
         libgfortran_versions = (3, 4, 5)
         other_libgfortran_version = libgfortran_versions[findfirst(v -> v != our_libgfortran_version.major, libgfortran_versions)]
-        @test_logs (:warn, Regex("but we are supposedly building for libgfortran$(other_libgfortran_version)")) (:warn, r"Linked library libgfortran.so.5") (:warn, r"Linked library libquadmath.so.0") (:warn, r"Linked library libgcc_s.so.1") readmeta(hello_world_path) do oh
-            p = deepcopy(platform)
-            p["libgfortran_version"] = "$(other_libgfortran_version).0.0"
-            @test !Auditor.audit(Prefix(testdir); platform=p, autofix=false)
+        @test_logs (:warn, Regex("but we are supposedly building for libgfortran$(other_libgfortran_version)")) (:warn, r"Linked library libgfortran.so.5") (:warn, r"Linked library libquadmath.so.0") (:warn, r"Linked library libgcc_s.so.1") readmeta(hello_world_path) do ohs
+            foreach(ohs) do oh
+                p = deepcopy(platform)
+                p["libgfortran_version"] = "$(other_libgfortran_version).0.0"
+                @test !Auditor.audit(Prefix(testdir); platform=p, autofix=false)
+            end
         end
     end
 end
