Add environment variable to select which benchmarkers to run. (#22139)

EMTEST_BENCHMARKERS accepts a comma separated list of benchmarkers to
run.

Author: brendandahl

site/source/docs/getting_started/test-suite.rst

@@ -181,11 +181,19 @@ As with all the test suites, you can also run a specific benchmark:
   # Run one specific benchmark
   test/runner benchmark.test_skinning
 
-Usually you will want to customize the in `test/test_benchmark.py` to
-run the benchmarks you want (there is currently no external config file). Things
-you may want to modify include:
+You can also specify which benchmarkers are run by using the environment
+variable `EMTEST_BENCHMARKERS`. It accepts a comma separated list of named
+benchmarkers (names can be found in `named_benchmarkers` in
+`test/test_benchmark.py`).
+
+.. code-block:: bash
+
+  # Run one specific benchmark and with clang and v8.
+  EMTEST_BENCHMARKERS=clang,v8 test/runner benchmark.test_skinning
+
+To further customize how the benchmarks are run, you will want to edit the file
+`test/test_benchmark.py`. Some of the options include:
 
-* ``benchmarkers`` is the list of VMs to run the benchmarks on.
 * ``DEFAULT_ARG`` is how long the benchmark should run (they all try to run for
   a similar amount of time for consistency).
 * ``TEST_REPS`` is how many times to repeat each run (more will take longer, but

test/common.py

@@ -63,7 +63,6 @@
 EMTEST_LACKS_NATIVE_CLANG = None
 EMTEST_VERBOSE = None
 EMTEST_REBASELINE = None
-EMTEST_FORCE64 = None
 
 # Verbosity level control for subprocess calls to configure + make.
 # 0: disabled.

test/runner.py

@@ -406,7 +406,6 @@ def configure():
   common.EMTEST_LACKS_NATIVE_CLANG = int(os.getenv('EMTEST_LACKS_NATIVE_CLANG', '0'))
   common.EMTEST_REBASELINE = int(os.getenv('EMTEST_REBASELINE', '0'))
   common.EMTEST_VERBOSE = int(os.getenv('EMTEST_VERBOSE', '0')) or shared.DEBUG
-  common.EMTEST_FORCE64 = int(os.getenv('EMTEST_FORCE64', '0'))
   if common.EMTEST_VERBOSE:
     logging.root.setLevel(logging.DEBUG)
 

test/test_benchmark.py

@@ -50,6 +50,10 @@
 
 LLVM_FEATURE_FLAGS = ['-mnontrapping-fptoint']
 
+# A comma separated list of benchmarkers to run during test_benchmark tests. See
+# `named_benchmarkers` for what is available.
+EMTEST_BENCHMARKERS = os.getenv('EMTEST_BENCHMARKERS', 'clang,v8,v8-lto,v8-ctors')
+
 
 class Benchmarker():
   # called when we init the object, which is during startup, even if we are
@@ -305,52 +309,30 @@ def get_output_files(self):
 
 benchmarkers: List[Benchmarker] = []
 
-if not common.EMTEST_FORCE64:
-  benchmarkers += [
-    NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]),
-    # NativeBenchmarker('gcc',   ['gcc', '-no-pie'],  ['g++', '-no-pie'])
-  ]
-
-if config.V8_ENGINE and config.V8_ENGINE in config.JS_ENGINES:
-  # avoid the baseline compiler running, because it adds a lot of noise
-  # (the nondeterministic time it takes to get to the full compiler ends up
-  # mattering as much as the actual benchmark)
-  aot_v8 = config.V8_ENGINE + ['--no-liftoff']
-  default_v8_name = os.environ.get('EMBENCH_NAME') or 'v8'
-  if common.EMTEST_FORCE64:
-    benchmarkers += [
-      EmscriptenBenchmarker(default_v8_name, aot_v8, ['-sMEMORY64=2']),
-    ]
-  else:
-    benchmarkers += [
-      EmscriptenBenchmarker(default_v8_name, aot_v8),
-      EmscriptenBenchmarker(default_v8_name + '-lto', aot_v8, ['-flto']),
-      EmscriptenBenchmarker(default_v8_name + '-ctors', aot_v8, ['-sEVAL_CTORS']),
-    ]
-  if os.path.exists(CHEERP_BIN):
-    benchmarkers += [
-      # CheerpBenchmarker('cheerp-v8-wasm', aot_v8),
-    ]
-
-if config.SPIDERMONKEY_ENGINE and config.SPIDERMONKEY_ENGINE in config.JS_ENGINES:
+# avoid the baseline compiler running, because it adds a lot of noise
+# (the nondeterministic time it takes to get to the full compiler ends up
+# mattering as much as the actual benchmark)
+aot_v8 = (config.V8_ENGINE if config.V8_ENGINE else []) + ['--no-liftoff']
+
+named_benchmarkers = {
+  'clang': NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]),
+  'gcc': NativeBenchmarker('gcc',   ['gcc', '-no-pie'],  ['g++', '-no-pie']),
+  'v8': EmscriptenBenchmarker('v8', aot_v8),
+  'v8-lto': EmscriptenBenchmarker('v8-lto', aot_v8, ['-flto']),
+  'v8-ctors': EmscriptenBenchmarker('v8-ctors', aot_v8, ['-sEVAL_CTORS']),
+  'v8-64': EmscriptenBenchmarker('v8-64', aot_v8, ['-sMEMORY64=2']),
+  'node': EmscriptenBenchmarker('node', config.NODE_JS),
+  'node-64': EmscriptenBenchmarker('node-64', config.NODE_JS, ['-sMEMORY64=2']),
+  'cherp-v8': CheerpBenchmarker('cheerp-v8-wasm', aot_v8),
   # TODO: ensure no baseline compiler is used, see v8
-  benchmarkers += [
-    # EmscriptenBenchmarker('sm', SPIDERMONKEY_ENGINE),
-  ]
-  if os.path.exists(CHEERP_BIN):
-    benchmarkers += [
-      # CheerpBenchmarker('cheerp-sm-wasm', SPIDERMONKEY_ENGINE),
-    ]
-
-if config.NODE_JS and config.NODE_JS in config.JS_ENGINES:
-  if common.EMTEST_FORCE64:
-    benchmarkers += [
-      EmscriptenBenchmarker('Node.js', config.NODE_JS, ['-sMEMORY64=2']),
-    ]
-  else:
-    benchmarkers += [
-      # EmscriptenBenchmarker('Node.js', config.NODE_JS),
-    ]
+  'sm': EmscriptenBenchmarker('sm', config.SPIDERMONKEY_ENGINE),
+  'cherp-sm': CheerpBenchmarker('cheerp-sm-wasm', config.SPIDERMONKEY_ENGINE)
+}
+
+for name in EMTEST_BENCHMARKERS.split(','):
+  if name not in named_benchmarkers:
+    raise Exception('error, unknown benchmarker ' + name)
+  benchmarkers.append(named_benchmarkers[name])
 
 
 class benchmark(common.RunnerCore):