Add main script: benchmark/runbench.jl

Use custom branch of `PkgBenchmark.jl` to support post-processing, which
we need.

Author: NHDaly

benchmark/Manifest.toml

@@ -0,0 +1,104 @@
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[BenchmarkTools]]
+deps = ["JSON", "Printf", "Statistics", "Test"]
+git-tree-sha1 = "e686f1754227e4748259f400839b83a1e8773e02"
+uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+version = "0.4.1"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[JSON]]
+deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"]
+git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.20.0"
+
+[[LibGit2]]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[Pkg]]
+deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[PkgBenchmark]]
+deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Pkg", "Printf", "ProgressMeter", "Random", "Test"]
+git-tree-sha1 = "d15f09c1e4a8c1075db6025fed97b8a852d5738e"
+repo-rev = "post-processing"
+repo-url = "https://github.com/NHDaly/PkgBenchmark.jl"
+uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+version = "0.2.0+"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[ProgressMeter]]
+deps = ["Distributed", "Printf", "Random", "Test"]
+git-tree-sha1 = "48058bc11607676e5bbc0b974af79106c6200787"
+uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
+version = "0.9.0"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[[Test]]
+deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

benchmark/Project.toml

@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"

benchmark/benchmarks.jl

@@ -7,16 +7,18 @@
 # It compares fixed-decimal types against the builtin Int and Float types of various sizes.
 # The output is written to a .csv file in the same directory as this file.
 
+# TODO: remove this file once BenchmarkTools has a built-in solution for diffing two
+# @benchmarkable runs
+using Pkg
+Pkg.activate(@__DIR__)
+
 using FixedPointDecimals
 using Random
 using BenchmarkTools, Statistics
 
-# # TODO: remove this file once BenchmarkTools has a built-in solution for diffing two
-# # @benchmarkable runs
-# include("subtract-benchmarks.jl")
-
 # Define a parent BenchmarkGroup to contain our suite
 const SUITE = BenchmarkGroup()
+const N = parse(Int, get(ENV, "BENCH_NUM_ITERS", "1000"))
 
 decimal_precision = 2
 
@@ -78,7 +80,7 @@ end
 for op in allops
     SUITE[opname(op)] = BenchmarkGroup()
     for T in alltypes
-        SUITE[opname(op)][type(T)] = BenchmarkGroup(["diff"])
+        SUITE[opname(op)][type(T)] = BenchmarkGroup(["base", "bench"])
     end
 end
 
@@ -88,7 +90,6 @@ for op in allops
     for T in alltypes
         print("$T ")
 
-        N = 1 # _000 #_000
         initial_value = zero(T)
         a = one(T)
 
@@ -98,10 +99,8 @@ for op in allops
 
         # Run the benchmark
         outbase = Ref(initial_value)
-        bbase = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
+        SUITE[opname(op)][type(T)]["base"] = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
         outbench = Ref(initial_value)
-        bbench = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
-        bdiff = bbench - bbase
-        SUITE[opname(op)][type(T)]["diff"] = bdiff
+        SUITE[opname(op)][type(T)]["bench"] = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
     end
 end

benchmark/runbench.jl

@@ -0,0 +1,51 @@
+module FixedPointDecimals_RunBench
+
+using Pkg
+
+Pkg.activate(@__DIR__)
+using PkgBenchmark, BenchmarkTools, Statistics
+
+const N = 1_000
+
+import Base: -, /
+function -(a::BenchmarkTools.TrialEstimate, b::BenchmarkTools.TrialEstimate)
+    ttol = max(params(a).time_tolerance, params(b).time_tolerance)
+    mtol = max(params(a).memory_tolerance, params(b).memory_tolerance)
+    p = BenchmarkTools.Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol)
+    return BenchmarkTools.TrialEstimate(p, -(time(a), time(b)), -(gctime(a), gctime(b)),
+                                        -(memory(a), memory(b)), -(allocs(a), allocs(b)))
+end
+function /(a::BenchmarkTools.TrialEstimate, b::Int)
+    ttol = params(a).time_tolerance / b
+    mtol = params(a).memory_tolerance / b
+    p = BenchmarkTools.Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol)
+    return BenchmarkTools.TrialEstimate(p, time(a)/b, gctime(a)/b,
+                                        memory(a)/b, allocs(a)/b)
+end
+
+function postprocess(results::BenchmarkGroup)
+    global _results = deepcopy(results)
+    for (op, op_group) in results.data
+        op_results = op_group.data
+        for (type, type_group) in op_results
+            benchresults = type_group.data
+            if op == "identity"
+                # For :identity, bench and base are identical so we don't want to subtract.
+                op_results[type] = median(benchresults["bench"]) / N
+            else
+                op_results[type] = median(benchresults["bench"])/N - median(benchresults["base"])/N
+            end
+        end
+    end
+    results
+end
+results = deepcopy(_results)
+((results)->(results["*"]["BigInt"]["bench"]=median(results["*"]["BigInt"]["bench"])/1000))(results)
+
+bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
+    benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
+end
+
+export_markdown("results.md", bench_results)
+
+end