[GR-26172] Add a benchmark-interface backend to run a fixed number of iterations

PullRequest: truffleruby/2020

Author: eregon

bench/benchmark-interface/lib/benchmark-interface.rb

@@ -12,6 +12,7 @@
 require 'benchmark-interface/frontends/mri'
 require 'benchmark-interface/backends/simple'
 require 'benchmark-interface/backends/stable'
+require 'benchmark-interface/backends/fixed-iterations'
 require 'benchmark-interface/backends/benchmark'
 require 'benchmark-interface/backends/bips'
 require 'benchmark-interface/backends/bench9000'

bench/benchmark-interface/lib/benchmark-interface/backends/fixed-iterations.rb

@@ -0,0 +1,48 @@
+# Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved. This
+# code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 2.0, or
+# GNU General Public License version 2, or
+# GNU Lesser General Public License version 2.1.
+
+module BenchmarkInterface
+  module Backends
+    module FixedIterations
+
+      def self.get_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def self.run(benchmark_set, names, options)
+        print_iterations = options['--iterations']
+        elapsed = options['--elapsed']
+        print_ips = options['--ips']
+
+        fixed_iterations = options['--fixed-iterations'].sort
+        total_iterations = fixed_iterations.last
+
+        benchmark_set.benchmarks(names).each do |benchmark|
+          puts benchmark.name
+          block = benchmark.block
+
+          next_iter = fixed_iterations.shift
+          start_time = get_time
+          (1..total_iterations).each do |iter|
+            block.call
+
+            if iter == next_iter
+              since_start = get_time - start_time
+              puts iter if print_iterations
+              puts since_start if elapsed
+              puts iter / since_start if print_ips
+
+              next_iter = fixed_iterations.shift
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

bench/benchmark-interface/lib/benchmark-interface/backends/simple.rb

@@ -61,8 +61,8 @@ def self.run(benchmark_set, names, options)
               # can print an ips and adjust for the next round to try to make
               # it take a second.
               ips = iterations / round_time
-              puts get_time - start_time if elapsed
               puts iterations if print_iterations
+              puts get_time - start_time if elapsed
               puts ips * inner_iterations
               iterations = (ips * freq).to_i
             end

bench/benchmark-interface/lib/benchmark-interface/backends/stable.rb

@@ -30,8 +30,8 @@ def self.run(benchmark_set, names, options)
             round_time = Time.now - start_round_time
 
             ips = 1 / round_time
-            puts Time.now - start_time if elapsed
             puts 1 if print_iterations
+            puts Time.now - start_time if elapsed
             puts ips * inner_iterations
           end
         end

bench/benchmark-interface/lib/benchmark-interface/run.rb

@@ -30,6 +30,8 @@ def self.run(args)
       '--freq' => 1,
       '--elapsed' => false,
       '--iterations' => false,
+      '--ips' => false,
+      '--fixed-iterations' => [],
       '--log' => nil,
       '--tag' => [],
       '--prop' => {}
@@ -49,6 +51,9 @@ def self.run(args)
             backend = BenchmarkInterface::Backends::Simple
           when '--stable'
             backend = BenchmarkInterface::Backends::Stable
+          when '--fixed-iterations'
+            backend = BenchmarkInterface::Backends::FixedIterations
+            options[arg] = args[n += 1].split(',').map { |iter| Integer(iter) }
           when '--bips'
             backend = BenchmarkInterface::Backends::Bips
           when '--bm'
@@ -72,6 +77,8 @@ def self.run(args)
             n += 1
           when '--elapsed'
             options[arg] = true
+          when '--ips'
+            options[arg] = true
           when '--log'
             options[arg] = args[n + 1]
             n += 1
@@ -148,9 +155,10 @@ def self.help
     puts '  list      list benchmarks available in the files'
     puts
     puts 'Backends:'
-    puts '  --simple'
+    puts '  --simple (default)'
     puts '  --stable'
-    puts '  --bips (default)'
+    puts '  --fixed-iterations ITERS1,ITERS2,...'
+    puts '  --bips'
     puts '  --bm'
     puts '  --bmbm'
     puts '  --bench9000'

mx.truffleruby/mx_truffleruby_benchmark.py

@@ -287,7 +287,7 @@ def runBenchmark(self, benchmark, bmSuiteArgs):
             lines = out.data.split('\n')[1:-1]
 
             data = self.filterLines(lines)
-            elapsed = [d for n, d in enumerate(data) if n % 3 == 0]
+            elapsed = [d for n, d in enumerate(data) if n % 3 == 1]
             samples = [d for n, d in enumerate(data) if n % 3 == 2]
 
             if lines[-1] == 'optimised away':