Collect test results into BuildKit-compatible JSON (#53145)

Author: tecosaur

.gitignore

@@ -39,6 +39,9 @@
 # Buildkite: Ignore the entire .buildkite directory
 /.buildkite
 
+# Builtkite: json test data
+/test/results.json
+
 # Buildkite: Ignore the unencrypted repo_key
 repo_key
 

test/buildkitetestjson.jl

@@ -0,0 +1,142 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Convert test(set) results to a Buildkit-compatible JSON representation.
+# Based on <https://buildkite.com/docs/test-analytics/importing-json#json-test-results-data-reference>.
+
+module BuildKiteTestJSON
+
+using Test
+using Dates
+
+export write_testset_json
+
+# Bootleg JSON writer
+
+"""
+    json_repr(io::IO, value; kwargs...) -> Nothing
+
+Obtain a JSON representation of `value`, and print it to `io`.
+
+This may not be the best, most feature-complete, or fastest implementation.
+However, it works for its intended purpose.
+"""
+function json_repr end
+
+function json_repr(io::IO, val::String; indent::Int=0)
+    print(io, '"')
+    escape_string(io, val, ('"',))
+    print(io, '"')
+end
+json_repr(io::IO, val::Integer; indent::Int=0) = print(io, val)
+json_repr(io::IO, val::Float64; indent::Int=0) = print(io, val)
+function json_repr(io::IO, val::Vector; indent::Int=0)
+    print(io, '[')
+    for elt in val
+        print(io, '\n', ' '^(indent + 2))
+        json_repr(io, elt; indent=indent+2)
+        elt === last(val) || print(io, ',')
+    end
+    print(io, '\n', ' '^indent, ']')
+end
+function json_repr(io::IO, val::Dict; indent::Int=0)
+    print(io, '{')
+    for (i, (k, v)) in enumerate(pairs(val))
+        print(io, '\n', ' '^(indent + 2))
+        json_repr(io, string(k))
+        print(io, ": ")
+        json_repr(io, v; indent=indent+2)
+        i === length(val) || print(io, ',')
+    end
+    print(io, '\n', ' '^indent, '}')
+end
+json_repr(io::IO, val::Any; indent::Int=0) = json_repr(io, string(val))
+
+# Test result processing
+
+function result_dict(testset::Test.DefaultTestSet, prefix::String="")
+    Dict{String, Any}(
+        "id" => Base.UUID(rand(UInt128)),
+        "scope" => join((prefix, testset.description), '/'),
+        "history" => if !isnothing(testset.time_end)
+            Dict{String, Any}(
+                "start_at" => testset.time_start,
+                "end_at" => testset.time_end,
+                "duration" => testset.time_end - testset.time_start)
+        else
+            Dict{String, Any}("start_at" => testset.time_start, "duration" => 0.0)
+        end)
+end
+
+function result_dict(result::Test.Result)
+    file, line = if !hasproperty(result, :source) || isnothing(result.source)
+        "unknown", 0
+    else
+        something(result.source.file, "unknown"), result.source.line
+    end
+    status = if result isa Test.Pass && result.test_type === :skipped
+        "skipped"
+    elseif result isa Test.Pass
+        "passed"
+    elseif result isa Test.Fail || result isa Test.Error
+        "failed"
+    else
+        "unknown"
+    end
+    data = Dict{String, Any}(
+        "name" => "$(result.test_type): $(result.orig_expr)",
+        "location" => string(file, ':', line),
+        "file_name" => file,
+        "result" => status)
+    add_failure_info!(data, result)
+end
+
+function add_failure_info!(data::Dict{String, Any}, result::Test.Result)
+    if result isa Test.Fail
+        data["failure_reason"] = if result.test_type === :test && !isnothing(result.data)
+            "Evaluated: $(result.data)"
+        elseif result.test_type === :test_throws_nothing
+            "No exception thrown"
+        elseif result.test_type === :test_throws_wrong
+            "Wrong exception type thrown"
+        else
+            "unknown"
+        end
+    elseif result isa Test.Error
+        data["failure_reason"] = if result.test_type === :test_error
+            if occursin("\nStacktrace:\n", result.backtrace)
+                err, trace = split(result.backtrace, "\nStacktrace:\n", limit=2)
+                data["failure_expanded"] = Dict{String, Any}(
+                    "expanded" => split(err, '\n'),
+                    "backtrace" => split(trace, '\n'))
+            end
+            "Exception (unexpectedly) thrown during test"
+        elseif result.test_type === :test_nonbool
+            "Expected the expression to evaluate to a Bool, not a $(typeof(result.data))"
+        elseif result.test_nonbool === :test_unbroken
+            "Expected this test to be broken, but it passed"
+        else
+            "unknown"
+        end
+    end
+    data
+end
+
+function collect_results!(results::Vector{Dict{String, Any}}, testset::Test.DefaultTestSet, prefix::String="")
+    common_data = result_dict(testset, prefix)
+    for (i, result) in enumerate(testset.results)
+        if result isa Test.Result
+            push!(results, merge(common_data, result_dict(result)))
+        elseif result isa Test.DefaultTestSet
+            collect_results!(results, result, common_data["scope"])
+        end
+    end
+    results
+end
+
+function write_testset_json(io::IO, testset::Test.DefaultTestSet)
+    data = Dict{String, Any}[]
+    collect_results!(data, testset)
+    json_repr(io, data)
+end
+
+end

test/runtests.jl

@@ -9,6 +9,9 @@ using Base: Experimental
 
 include("choosetests.jl")
 include("testenv.jl")
+include("buildkitetestjson.jl")
+
+using .BuildKiteTestJSON
 
 (; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS)
 tests = unique(tests)
@@ -424,6 +427,10 @@ cd(@__DIR__) do
         Test.record(o_ts, fake)
         Test.pop_testset()
     end
+
+    Base.get_bool_env("CI", false) &&
+        open(io -> write_testset_json(io, o_ts), "results.json", "w")
+
     Test.TESTSET_PRINT_ENABLE[] = true
     println()
     # o_ts.verbose = true # set to true to show all timings when successful