Merge branch 'andriy/run-652-refactor-ee-benchmarks' into 'master'

RUN-652 Refactor EE benchmarks

This change is a dev experience improvement. As the language server still uses `cargo` in most of the cases, this MR fixes the EE benchmarks to be visible for the `cargo`.

The `bazel` is still the preferable way to run the benchmarks, just like before. But now the `rust-analyzer` know about the benchmarks, and hence in VS Code we see all the types etc...

The main changes:

1. Move all the benchmarks under `benches` directory, so the `rust-analyzer` can see them.
2. Move the common benchmarking code under `benches/lib`.
3. Add a warning if there are no sandbox environment variables defined. 

See merge request dfinity-lab/public/ic!12650

Author: berestovskyy

Cargo.lock

@@ -104,6 +104,7 @@ members = [
   "rs/elastic_common_schema",
   "rs/embedders",
   "rs/execution_environment",
+  "rs/execution_environment/benches/lib",
   "rs/guestos_vsock_agent",
   "rs/http_endpoints/public",
   "rs/http_endpoints/metrics",

Cargo.toml

@@ -160,9 +160,9 @@ BENCH_DEPENDENCIES = [
 rust_library(
     name = "execution_environment_bench",
     srcs = [
-        "benchmarks/common.rs",
-        "benchmarks/common_wat.rs",
-        "benchmarks/lib.rs",
+        "benches/lib/src/common.rs",
+        "benches/lib/src/lib.rs",
+        "benches/lib/src/wat.rs",
     ],
     aliases = ALIASES,
     proc_macro_deps = MACRO_DEPENDENCIES,
@@ -172,31 +172,31 @@ rust_library(
 
 rust_bench(
     name = "execute_inspect_message_bench",
-    srcs = ["benchmarks/execute_inspect_message.rs"],
+    srcs = ["benches/execute_inspect_message.rs"],
     data = DATA,
     env = ENV,
     deps = [":execution_environment_bench"] + BENCH_DEPENDENCIES,
 )
 
 rust_bench(
     name = "execute_query_bench",
-    srcs = ["benchmarks/execute_query.rs"],
+    srcs = ["benches/execute_query.rs"],
     data = DATA,
     env = ENV,
     deps = [":execution_environment_bench"] + BENCH_DEPENDENCIES,
 )
 
 rust_bench(
     name = "execute_update_bench",
-    srcs = ["benchmarks/execute_update.rs"],
+    srcs = ["benches/execute_update.rs"],
     data = DATA,
     env = ENV,
     deps = [":execution_environment_bench"] + BENCH_DEPENDENCIES,
 )
 
 rust_bench(
     name = "scheduler_bench",
-    srcs = ["benchmarks/scheduler.rs"],
+    srcs = ["benches/scheduler.rs"],
     data = DATA,
     env = ENV,
     deps = [
@@ -209,7 +209,7 @@ rust_bench(
 
 rust_bench(
     name = "100k_canisters_bench",
-    srcs = ["benchmarks/100k_canisters.rs"],
+    srcs = ["benches/100k_canisters.rs"],
     data = DATA + ["//rs/rust_canisters/canister_creator:canister_creator_canister"],
     env = dict(ENV.items() + [
         ("CANISTER_CREATOR_CANISTER_WASM_PATH", "$(rootpath //rs/rust_canisters/canister_creator:canister_creator_canister)"),

rs/execution_environment/BUILD.bazel

@@ -1,7 +1,7 @@
 [package]
+edition = "2021"
 name = "ic-execution-environment"
 version = "0.8.0"
-edition = "2021"
 
 [dependencies]
 candid = "0.8.1"
@@ -41,20 +41,22 @@ nix = "0.23.0"
 num-rational = "0.2.2"
 num-traits = "0.2.12"
 phantom_newtype = { path = "../phantom_newtype" }
-prometheus = { version = "0.12.0", features = [ "process" ] }
+prometheus = { version = "0.12.0", features = ["process"] }
 rand = "0.8"
 scoped_threadpool = "0.1.*"
-serde = { version = "1.0.99", features = [ "derive" ] }
+serde = { version = "1.0.99", features = ["derive"] }
 serde_cbor = "0.11.1"
 slog = { version = "2.5.2", features = ["nested-values", "release_max_level_debug"] }
 strum = "0.23.0"
 threadpool = "1.8.1"
-tokio = { version = "1.17.0", features = ["sync", "rt"] }
-tower = { version = "0.4.11", features = ["limit", "buffer", "timeout"] }
+tokio = { version = "1.17.0", features = ["rt", "sync"] }
+tower = { version = "0.4.11", features = ["buffer", "limit", "timeout"] }
 
 [dev-dependencies]
 assert_matches = "1.3.0"
-criterion = { version = "0.3", features = ["html_reports"] }
+canister-test = { path = "../rust_canisters/canister_test" }
+criterion = { version = "0.5", features = ["html_reports"] }
+execution-environment-bench = { path = "benches/lib" }
 iai = "0.1"
 ic-btc-test-utils = { git = "https://github.com/dfinity/bitcoin-canister", rev = "e4e89f2caedffbe0cfdec6f9d4a77f66dcb9119e" }
 ic-crypto-sha = { path = "../crypto/sha" }
@@ -83,5 +85,25 @@ escargot = "0.5"
 default = []
 sigsegv_handler_checksum = [
 	"ic-canister-sandbox-replica-controller/sigsegv_handler_checksum",
-	"memory_tracker/sigsegv_handler_checksum"
+	"memory_tracker/sigsegv_handler_checksum",
 ]
+
+[[bench]]
+harness = false
+name = "100k_canisters"
+
+[[bench]]
+harness = false
+name = "execute_inspect_message"
+
+[[bench]]
+harness = false
+name = "execute_query"
+
+[[bench]]
+harness = false
+name = "execute_update"
+
+[[bench]]
+harness = false
+name = "scheduler"

rs/execution_environment/Cargo.toml

@@ -2,19 +2,17 @@
 /// Benchmark System API performance in `execute_inspect_message()`
 ///
 use criterion::{criterion_group, criterion_main, Criterion};
-use execution_environment_bench::common;
-use execution_environment_bench::common_wat::*;
+use execution_environment_bench::{common, wat::*};
 use ic_execution_environment::execution::inspect_message;
 
 use ic_execution_environment::ExecutionEnvironment;
 use ic_logger::replica_logger::no_op_logger;
 use ic_test_utilities::types::ids::user_test_id;
 use ic_test_utilities::types::messages::SignedIngressBuilder;
-use lazy_static::lazy_static;
 
-lazy_static! {
-    /// List of benchmarks: benchmark id (name), WAT, expected instructions.
-    pub static ref BENCHMARKS: Vec<common::Benchmark> = vec![
+pub fn execute_inspect_message_bench(c: &mut Criterion) {
+    // List of benchmarks: benchmark id (name), WAT, expected instructions.
+    let benchmarks: Vec<common::Benchmark> = vec![
         common::Benchmark(
             "ic0_msg_method_name_size()",
             Module::InspectMessage.from_ic0("msg_method_name_size", NoParams, Result::I32),
@@ -36,13 +34,10 @@ lazy_static! {
             1,
         ),
     ];
-}
-
-pub fn bench_execute_inspect_message(c: &mut Criterion) {
     common::run_benchmarks(
         c,
         "inspect",
-        &BENCHMARKS,
+        &benchmarks,
         |exec_env: &ExecutionEnvironment,
          expected_instructions,
          common::BenchmarkArgs {
@@ -82,5 +77,5 @@ pub fn bench_execute_inspect_message(c: &mut Criterion) {
     );
 }
 
-criterion_group!(benchmarks, bench_execute_inspect_message);
+criterion_group!(benchmarks, execute_inspect_message_bench);
 criterion_main!(benchmarks);