[GR-26172] Add asciidoctor warmup benchmark

PullRequest: truffleruby/2030

Author: eregon

bench/asciidoctor/asciidoctor-convert.rb

@@ -0,0 +1,9 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_text = File.read(sample_file)
+document = Asciidoctor.load(sample_file_text)
+
+benchmark 'convert' do
+  document.convert
+end

bench/asciidoctor/asciidoctor-load-file.rb

@@ -0,0 +1,9 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_io = File.open(sample_file)
+
+benchmark 'load-file' do
+  Asciidoctor.load sample_file_io
+  sample_file_io.seek 0
+end

bench/asciidoctor/asciidoctor-load-string.rb

@@ -0,0 +1,8 @@
+require_relative 'asciidoctor/lib/asciidoctor'
+
+sample_file = File.expand_path('../data/userguide.adoc', __FILE__)
+sample_file_text = File.read(sample_file)
+
+benchmark 'load-string' do
+  Asciidoctor.load sample_file_text
+end

bench/asciidoctor/asciidoctor.rb

@@ -28,15 +28,6 @@
   end
 end
 
-benchmark 'load-string' do
-  Asciidoctor.load sample_file_text
-end
-
-benchmark 'load-file' do
-  Asciidoctor.load sample_file_io
-  sample_file_io.seek 0
-end
-
 document = Asciidoctor.load(sample_file_text)
 converted = document.convert
 lines = converted.split('\n')
@@ -81,7 +72,3 @@ def document
 benchmark 'join-lines' do
   lines * '\n'
 end
-
-benchmark 'convert' do
-  document.convert
-end

ci.jsonnet

@@ -440,6 +440,7 @@ local part_definitions = {
     asciidoctor: { benchmarks+:: ["asciidoctor"] },
     other_extra: { benchmarks+:: ["savina"] },
     other: { benchmarks+:: ["micro", "image-demo", "optcarrot", "synthetic", "rubykon", "liquid"] },
+    warmup: { benchmarks+:: ["ruby-warmup"] },
 
     server: {
       local build = self,
@@ -608,6 +609,17 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-benchmarks-asciidoctor-svm-graal-core-multi-tier": shared + asciidoctor + svm_configurations["svm-graal-core"] + $.use.multi_tier,
       "ruby-benchmarks-asciidoctor-svm-graal-enterprise": shared + asciidoctor + svm_configurations["svm-graal-enterprise"],
       "ruby-benchmarks-asciidoctor-svm-graal-enterprise-multi-tier": shared + asciidoctor + svm_configurations["svm-graal-enterprise"] + $.use.multi_tier,
+      local warmup = $.benchmark.runner + $.benchmark.warmup + { timelimit: "00:55:00" },
+      "ruby-benchmarks-warmup-mri": shared + warmup + other_rubies.mri,
+      "ruby-benchmarks-warmup-jruby": shared + warmup + other_rubies.jruby,
+      "ruby-benchmarks-warmup-graal-core": shared + warmup + graal_configurations["graal-core"],
+      "ruby-benchmarks-warmup-graal-core-multi-tier": shared + warmup + graal_configurations["graal-core"] + $.use.multi_tier,
+      "ruby-benchmarks-warmup-graal-enterprise": shared + warmup + graal_configurations["graal-enterprise"],
+      "ruby-benchmarks-warmup-graal-enterprise-multi-tier": shared + warmup + graal_configurations["graal-enterprise"] + $.use.multi_tier,
+      "ruby-benchmarks-warmup-svm-graal-core": shared + warmup + svm_configurations["svm-graal-core"],
+      "ruby-benchmarks-warmup-svm-graal-core-multi-tier": shared + warmup + svm_configurations["svm-graal-core"] + $.use.multi_tier,
+      "ruby-benchmarks-warmup-svm-graal-enterprise": shared + warmup + svm_configurations["svm-graal-enterprise"],
+      "ruby-benchmarks-warmup-svm-graal-enterprise-multi-tier": shared + warmup + svm_configurations["svm-graal-enterprise"] + $.use.multi_tier,
 
       local other = $.benchmark.runner + $.benchmark.other + $.benchmark.other_extra + { timelimit: "00:40:00" },
       local svm_other = $.benchmark.runner + $.benchmark.other + { timelimit: "01:00:00" },

mx.truffleruby/mx_truffleruby_benchmark.py

@@ -245,6 +245,9 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
         } for region, region_data in data.items() for n, sample in enumerate(region_data['samples'])]
 
 class AllBenchmarksBenchmarkSuite(RubyBenchmarkSuite):
+    def config(self):
+        return {'kind': 'simple'}
+
     def benchmarkList(self, bmSuiteArgs):
         raise NotImplementedError()
 
@@ -261,24 +264,48 @@ def filterLines(self, lines):
         data = []
         for line in lines:
             try:
-                data.append(float(line))
+                data.append(int(line))
             except ValueError:
-                mx.log_error(line)
+                try:
+                    data.append(float(line))
+                except ValueError:
+                    mx.log_error(line)
         if len(data) % 3 != 0:
             raise AssertionError("Number of values not a multiple of 3")
         return data
 
     def runBenchmark(self, benchmark, bmSuiteArgs):
+        directory = self.directory()
+        if directory is None:
+            directory, benchmark = benchmark.split('/')
+
         arguments = ['benchmark']
-        arguments.extend(['--simple', '--elapsed', '--iterations'])
-        arguments.extend(['--time', str(self.time())])
+        if self.config()['kind'] == 'simple':
+            arguments.extend(['--simple', '--elapsed', '--iterations'])
+            time = self.time()
+            if isinstance(time, dict):
+                if benchmark in time:
+                    time = str(time[benchmark])
+                else:
+                    time = str(time['default'])
+            else:
+                time = str(self.time())
+            arguments.extend(['--time', time])
+        elif self.config()['kind'] == 'fixed-iterations':
+            iterations_arg = ','.join([str(i) for i in sorted(self.config()['iterations'][benchmark].keys())])
+            arguments.extend(['--elapsed', '--iterations', '--ips'])
+            arguments.extend(['--fixed-iterations'])
+            arguments.extend([iterations_arg])
+        else:
+            raise AssertionError("Unknown benchmark kind: " + self.config()['kind'])
+
         if ':' in benchmark:
             benchmark_file, benchmark_name = benchmark.split(':')
             benchmark_names = [benchmark_name]
         else:
             benchmark_file = benchmark
             benchmark_names = []
-        arguments.extend(['bench/' + self.directory() + '/' + benchmark_file + '.rb'])
+        arguments.extend(['bench/' + directory + '/' + benchmark_file + '.rb'])
         arguments.extend(benchmark_names)
         arguments.extend(bmSuiteArgs)
         out = mx.OutputCapture()
@@ -287,6 +314,7 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
             lines = out.data.split('\n')[1:-1]
 
             data = self.filterLines(lines)
+            iterations = [d for n, d in enumerate(data) if n % 3 == 0]
             elapsed = [d for n, d in enumerate(data) if n % 3 == 1]
             samples = [d for n, d in enumerate(data) if n % 3 == 2]
 
@@ -311,6 +339,16 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
                     'extra.metric.elapsed-num': elapsed[-1] + 2.0 if elapsed else 2.0, # just put the data point beyond the last one a bit
                     'error': 'optimised away'
                 }]
+            elif self.config()['kind'] == 'fixed-iterations':
+                iteration_config = self.config()['iterations'][benchmark]
+                return [{
+                    'benchmark': benchmark,
+                    'metric.name': iteration_config[iteration],
+                    'metric.iteration': iteration,
+                    'metric.value': e,
+                    'metric.unit': 's',
+                    'metric.better': 'lower'
+                } for n, (e, iteration) in enumerate(zip(elapsed, iterations)) if iteration in iteration_config]
             else:
                 return [{
                     'benchmark': benchmark,
@@ -469,15 +507,20 @@ def time(self):
     'asciidoctor:string-lines',
     'asciidoctor:read-line',
     'asciidoctor:restore-line',
-    'asciidoctor:load-string',
-    'asciidoctor:load-file',
     'asciidoctor:quote-match',
     'asciidoctor:quote-sub',
     'asciidoctor:join-lines',
-    'asciidoctor:convert'
+    'asciidoctor-convert',
+    'asciidoctor-load-file',
+    'asciidoctor-load-string'
 ]
 
-asciidoctor_benchmark_time = 120
+asciidoctor_benchmark_time = {
+    'asciidoctor-convert': 400,
+    'asciidoctor-load-file': 400,
+    'asciidoctor-load-string': 400,
+    'default': 120
+}
 
 class AsciidoctorBenchmarkSuite(AllBenchmarksBenchmarkSuite):
     def name(self):
@@ -642,6 +685,32 @@ def benchmarkList(self, bmSuiteArgs):
     def time(self):
         return 60
 
+warmup_benchmarks = [
+    'asciidoctor/asciidoctor-convert',
+    'asciidoctor/asciidoctor-load-file',
+    'asciidoctor/asciidoctor-load-string',
+    'rubykon/rubykon',
+]
+
+class WarmupBenchmarkSuite(AllBenchmarksBenchmarkSuite):
+    def config(self):
+        iterations = {
+            'asciidoctor-convert':     {10:'startup', 100:'early-warmup', 500:'late-warmup'},
+            'asciidoctor-load-file':   {10:'startup', 100:'early-warmup', 500:'late-warmup'},
+            'asciidoctor-load-string': {10:'startup', 100:'early-warmup', 500:'late-warmup'},
+            'rubykon':                 {1:'startup', 10:'early-warmup', 30:'late-warmup'},
+        }
+        return {'kind': 'fixed-iterations', 'iterations': iterations}
+
+    def name(self):
+        return 'ruby-warmup'
+
+    def directory(self):
+        return None
+
+    def benchmarkList(self, bmSuiteArgs):
+        return warmup_benchmarks
+
 mx_benchmark.add_bm_suite(BuildStatsBenchmarkSuite())
 mx_benchmark.add_bm_suite(AllocationBenchmarkSuite())
 mx_benchmark.add_bm_suite(InstructionsBenchmarkSuite())
@@ -660,3 +729,4 @@ def time(self):
 mx_benchmark.add_bm_suite(ServerBenchmarkSuite())
 mx_benchmark.add_bm_suite(RubykonBenchmarkSuite())
 mx_benchmark.add_bm_suite(LiquidBenchmarkSuite())
+mx_benchmark.add_bm_suite(WarmupBenchmarkSuite())