Add asciidoctor warmup benchmark

Author: bjfish

bench/asciidoctor/asciidoctor-convert.rb

@@ -0,0 +1,9 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_text = File.read(sample_file)
+document = Asciidoctor.load(sample_file_text)
+
+benchmark 'convert' do
+  document.convert
+end

bench/asciidoctor/asciidoctor-load-file.rb

@@ -0,0 +1,9 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_io = File.open(sample_file)
+
+benchmark 'load-file' do
+  Asciidoctor.load sample_file_io
+  sample_file_io.seek 0
+end

bench/asciidoctor/asciidoctor-load-string.rb

@@ -0,0 +1,8 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_text = File.read(sample_file)
+
+benchmark 'load-string' do
+  Asciidoctor.load sample_file_text
+end

bench/asciidoctor/asciidoctor.rb

@@ -28,15 +28,6 @@
   end
 end
 
-benchmark 'load-string' do
-  Asciidoctor.load sample_file_text
-end
-
-benchmark 'load-file' do
-  Asciidoctor.load sample_file_io
-  sample_file_io.seek 0
-end
-
 document = Asciidoctor.load(sample_file_text)
 converted = document.convert
 lines = converted.split('\n')
@@ -81,7 +72,3 @@ def document
 benchmark 'join-lines' do
   lines * '\n'
 end
-
-benchmark 'convert' do
-  document.convert
-end

ci.jsonnet

@@ -444,6 +444,7 @@ local part_definitions = {
     chunky: { benchmarks+:: ["chunky"] },
     psd: { benchmarks+:: ["psd"] },
     asciidoctor: { benchmarks+:: ["asciidoctor"] },
+    asciidoctor_warmup: { benchmarks+:: ["asciidoctor-warmup"] },
     other_extra: { benchmarks+:: ["savina"] },
     other: { benchmarks+:: ["micro", "image-demo", "optcarrot", "synthetic", "rubykon", "liquid"] },
 
@@ -614,6 +615,17 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-benchmarks-asciidoctor-svm-graal-core-multi-tier": shared + asciidoctor + svm_configurations["svm-graal-core"] + $.use.multi_tier,
       "ruby-benchmarks-asciidoctor-svm-graal-enterprise": shared + asciidoctor + svm_configurations["svm-graal-enterprise"],
       "ruby-benchmarks-asciidoctor-svm-graal-enterprise-multi-tier": shared + asciidoctor + svm_configurations["svm-graal-enterprise"] + $.use.multi_tier,
+      local asciidoctor_warmup = $.benchmark.runner + $.benchmark.asciidoctor_warmup + { timelimit: "00:55:00" },
+      "ruby-benchmarks-asciidoctor-mri-warmup": shared + asciidoctor_warmup + other_rubies.mri,
+      "ruby-benchmarks-asciidoctor-jruby-warmup": shared + asciidoctor_warmup + other_rubies.jruby,
+      "ruby-benchmarks-asciidoctor-graal-core-warmup": shared + asciidoctor_warmup + graal_configurations["graal-core"],
+      "ruby-benchmarks-asciidoctor-graal-core-multi-tier-warmup": shared + asciidoctor_warmup + graal_configurations["graal-core"] + $.use.multi_tier,
+      "ruby-benchmarks-asciidoctor-graal-enterprise-warmup": shared + asciidoctor_warmup + graal_configurations["graal-enterprise"],
+      "ruby-benchmarks-asciidoctor-graal-enterprise-multi-tier-warmup": shared + asciidoctor_warmup + graal_configurations["graal-enterprise"] + $.use.multi_tier,
+      "ruby-benchmarks-asciidoctor-svm-graal-core-warmup": shared + asciidoctor_warmup + svm_configurations["svm-graal-core"],
+      "ruby-benchmarks-asciidoctor-svm-graal-core-multi-tier-warmup": shared + asciidoctor_warmup + svm_configurations["svm-graal-core"] + $.use.multi_tier,
+      "ruby-benchmarks-asciidoctor-svm-graal-enterprise-warmup": shared + asciidoctor_warmup + svm_configurations["svm-graal-enterprise"],
+      "ruby-benchmarks-asciidoctor-svm-graal-enterprise-multi-tier-warmup": shared + asciidoctor_warmup + svm_configurations["svm-graal-enterprise"] + $.use.multi_tier,
 
       local other = $.benchmark.runner + $.benchmark.other + $.benchmark.other_extra + { timelimit: "00:40:00" },
       local svm_other = $.benchmark.runner + $.benchmark.other + { timelimit: "01:00:00" },

mx.truffleruby/mx_truffleruby_benchmark.py

@@ -245,6 +245,9 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
         } for region, region_data in data.items() for n, sample in enumerate(region_data['samples'])]
 
 class AllBenchmarksBenchmarkSuite(RubyBenchmarkSuite):
+    def config(self):
+        return {'kind': 'simple'}
+
     def benchmarkList(self, bmSuiteArgs):
         raise NotImplementedError()
 
@@ -270,8 +273,25 @@ def filterLines(self, lines):
 
     def runBenchmark(self, benchmark, bmSuiteArgs):
         arguments = ['benchmark']
-        arguments.extend(['--simple', '--elapsed', '--iterations'])
-        arguments.extend(['--time', str(self.time())])
+        if self.config()['kind'] == 'simple':
+            arguments.extend(['--simple', '--elapsed', '--iterations'])
+            time = self.time()
+            if isinstance(time, dict):
+                if benchmark in time:
+                    time = str(time[benchmark])
+                else:
+                    time = str(time['default'])
+            else:
+                time = str(self.time())
+            arguments.extend(['--time', time])
+        elif self.config()['kind'] == 'fixed-iterations':
+            iterations_arg = ','.join([str(i) for i in sorted(self.config()['iterations'][benchmark].keys())])
+            arguments.extend(['--elapsed', '--iterations', '--ips'])
+            arguments.extend(['--fixed-iterations'])
+            arguments.extend([iterations_arg])
+        else:
+            raise AssertionError("Unknown benchmark kind: " + self.config()['kind'])
+
         if ':' in benchmark:
             benchmark_file, benchmark_name = benchmark.split(':')
             benchmark_names = [benchmark_name]
@@ -287,6 +307,7 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
             lines = out.data.split('\n')[1:-1]
 
             data = self.filterLines(lines)
+            iterations = [d for n, d in enumerate(data) if n % 3 == 0]
             elapsed = [d for n, d in enumerate(data) if n % 3 == 1]
             samples = [d for n, d in enumerate(data) if n % 3 == 2]
 
@@ -311,6 +332,15 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
                     'extra.metric.elapsed-num': elapsed[-1] + 2.0 if elapsed else 2.0, # just put the data point beyond the last one a bit
                     'error': 'optimised away'
                 }]
+            elif self.config()['kind'] == 'fixed-iterations':
+                iteration_config = self.config()['iterations'][benchmark]
+                return [{
+                    'benchmark': benchmark,
+                    'metric.name': iteration_config[iteration],
+                    'metric.value': e,
+                    'metric.unit': 's',
+                    'metric.better': 'lower'
+                } for n, (e, iteration) in enumerate(zip(elapsed, iterations)) if iteration in iteration_config]
             else:
                 return [{
                     'benchmark': benchmark,
@@ -469,15 +499,15 @@ def time(self):
     'asciidoctor:string-lines',
     'asciidoctor:read-line',
     'asciidoctor:restore-line',
-    'asciidoctor:load-string',
-    'asciidoctor:load-file',
     'asciidoctor:quote-match',
     'asciidoctor:quote-sub',
     'asciidoctor:join-lines',
-    'asciidoctor:convert'
+    'asciidoctor-convert',
+    'asciidoctor-load-file',
+    'asciidoctor-load-string'
 ]
 
-asciidoctor_benchmark_time = 120
+asciidoctor_benchmark_time = {'asciidoctor-convert':400, 'asciidoctor-load-file':400, 'asciidoctor-load-string':400, 'default': 120}
 
 class AsciidoctorBenchmarkSuite(AllBenchmarksBenchmarkSuite):
     def name(self):
@@ -642,6 +672,28 @@ def benchmarkList(self, bmSuiteArgs):
     def time(self):
         return 60
 
+asciidoctor_warmup_benchmarks = [
+    'asciidoctor-convert',
+    'asciidoctor-load-file',
+    'asciidoctor-load-string'
+]
+
+class AsciidoctorWarmupBenchmarkSuite(AllBenchmarksBenchmarkSuite):
+    def config(self):
+        iterations = {'asciidoctor-convert':     {10:'startup', 20:'early-warmup', 100:'late-warmup'},
+                      'asciidoctor-load-file':   {10:'startup', 100:'early-warmup', 500:'late-warmup'},
+                      'asciidoctor-load-string': {10:'startup', 100:'early-warmup', 500:'late-warmup'}}
+        return {'kind': 'fixed-iterations', 'iterations': iterations}
+
+    def name(self):
+        return 'asciidoctor-warmup'
+
+    def directory(self):
+        return 'asciidoctor'
+
+    def benchmarkList(self, bmSuiteArgs):
+        return asciidoctor_warmup_benchmarks
+
 mx_benchmark.add_bm_suite(BuildStatsBenchmarkSuite())
 mx_benchmark.add_bm_suite(AllocationBenchmarkSuite())
 mx_benchmark.add_bm_suite(InstructionsBenchmarkSuite())
@@ -660,3 +712,4 @@ def time(self):
 mx_benchmark.add_bm_suite(ServerBenchmarkSuite())
 mx_benchmark.add_bm_suite(RubykonBenchmarkSuite())
 mx_benchmark.add_bm_suite(LiquidBenchmarkSuite())
+mx_benchmark.add_bm_suite(AsciidoctorWarmupBenchmarkSuite())