From a65f0bd1d8c4da5c855153e0bd697d09a086b195 Mon Sep 17 00:00:00 2001 From: Aner Ben Efraim Date: Tue, 23 Jul 2024 11:28:27 +0300 Subject: [PATCH 1/2] test: regression test for all regression files --- .../src/tests/regression_tests.rs | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/committer_cli/src/tests/regression_tests.rs b/crates/committer_cli/src/tests/regression_tests.rs index fa0b1baf..0768ed96 100644 --- a/crates/committer_cli/src/tests/regression_tests.rs +++ b/crates/committer_cli/src/tests/regression_tests.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, fs}; use committer::{ block_committer::input::{ConfigImpl, Input}, @@ -169,3 +169,45 @@ pub async fn test_regression_committer_flow() { // Assert the execution time does not exceed the threshold. assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST); } + +#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] +#[tokio::test(flavor = "multi_thread")] +pub async fn test_regression_committer_all_files() { + let dir_path = fs::read_dir("./benches/regression_files").unwrap(); + let mut file_counter = 0; + for file_path in dir_path { + file_counter += 1; + let CommitterRegressionInput { + committer_input, + contract_states_root: expected_contract_states_root, + contract_classes_root: expected_contract_classes_root, + expected_facts, + } = serde_json::from_str(&fs::read_to_string(file_path.unwrap().path()).unwrap()).unwrap(); + + let start = std::time::Instant::now(); + // Benchmark the committer flow test. + commit(committer_input.0, OUTPUT_PATH.to_owned()).await; + let execution_time = std::time::Instant::now() - start; + + // Assert correctness of the output of the committer flow test. + let CommitterRegressionOutput { + contract_storage_root_hash, + compiled_class_root_hash, + storage: + StorageObject { + storage: Value::Object(storage_changes), + }, + } = serde_json::from_str(&std::fs::read_to_string(OUTPUT_PATH).unwrap()).unwrap() + else { + panic!("Expected the storage to be an object."); + }; + + assert_eq!(contract_storage_root_hash, expected_contract_states_root); + assert_eq!(compiled_class_root_hash, expected_contract_classes_root); + assert_eq!(storage_changes, *expected_facts); + + // Assert the execution time does not exceed the threshold. + assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST); + } + assert_eq!(file_counter, 100); +} From 9bb981a4a8ff4d2b791919cf996386dc7fc91c16 Mon Sep 17 00:00:00 2001 From: Aner Ben Efraim Date: Tue, 23 Jul 2024 17:11:16 +0300 Subject: [PATCH 2/2] refactor: extract assert committer flow function --- .../src/tests/regression_tests.rs | 64 +++++++------------ 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/crates/committer_cli/src/tests/regression_tests.rs b/crates/committer_cli/src/tests/regression_tests.rs index 0768ed96..f6f1d1b5 100644 --- a/crates/committer_cli/src/tests/regression_tests.rs +++ b/crates/committer_cli/src/tests/regression_tests.rs @@ -22,6 +22,7 @@ const MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST: f64 = 5.0; const SINGLE_TREE_FLOW_INPUT: &str = include_str!("../../benches/tree_flow_inputs.json"); const FLOW_TEST_INPUT: &str = include_str!("../../benches/committer_flow_inputs.json"); const OUTPUT_PATH: &str = "benchmark_output.txt"; +const EXPECTED_NUMBER_OF_FILES: usize = 100; #[derive(derive_more::Deref)] struct FactMap(Map); @@ -100,7 +101,7 @@ impl<'de> Deserialize<'de> for TreeRegressionInput { } } -#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] +#[ignore = "To avoid running the regression test in Coverage or without the --release flag."] #[tokio::test(flavor = "multi_thread")] pub async fn test_regression_single_tree() { let TreeRegressionInput { @@ -135,19 +136,17 @@ pub async fn test_regression_single_tree() { assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_SINGLE_TREE_BECHMARK_TEST); } -#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] -#[tokio::test(flavor = "multi_thread")] -pub async fn test_regression_committer_flow() { +pub async fn test_single_committer_flow(input: &str, output_path: &str) { let CommitterRegressionInput { committer_input, contract_states_root: expected_contract_states_root, contract_classes_root: expected_contract_classes_root, expected_facts, - } = serde_json::from_str(FLOW_TEST_INPUT).unwrap(); + } = serde_json::from_str(input).unwrap(); let start = std::time::Instant::now(); // Benchmark the committer flow test. - commit(committer_input.0, OUTPUT_PATH.to_owned()).await; + commit(committer_input.0, output_path.to_owned()).await; let execution_time = std::time::Instant::now() - start; // Assert correctness of the output of the committer flow test. @@ -157,7 +156,7 @@ pub async fn test_regression_committer_flow() { storage: StorageObject { storage: Value::Object(storage_changes), }, - } = serde_json::from_str(&std::fs::read_to_string(OUTPUT_PATH).unwrap()).unwrap() + } = serde_json::from_str(&std::fs::read_to_string(output_path).unwrap()).unwrap() else { panic!("Expected the storage to be an object."); }; @@ -169,45 +168,26 @@ pub async fn test_regression_committer_flow() { // Assert the execution time does not exceed the threshold. assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST); } +#[ignore = "To avoid running the regression test in Coverage or without the --release flag."] +#[tokio::test(flavor = "multi_thread")] +pub async fn test_regression_committer_flow() { + test_single_committer_flow(FLOW_TEST_INPUT, OUTPUT_PATH).await; +} -#[ignore = "To avoid running the benchmark test in Coverage or without the --release flag."] +#[ignore = "To avoid running the regression test in Coverage or without the --release flag."] #[tokio::test(flavor = "multi_thread")] pub async fn test_regression_committer_all_files() { + assert_eq!( + fs::read_dir("./benches/regression_files").unwrap().count(), + EXPECTED_NUMBER_OF_FILES + ); let dir_path = fs::read_dir("./benches/regression_files").unwrap(); - let mut file_counter = 0; for file_path in dir_path { - file_counter += 1; - let CommitterRegressionInput { - committer_input, - contract_states_root: expected_contract_states_root, - contract_classes_root: expected_contract_classes_root, - expected_facts, - } = serde_json::from_str(&fs::read_to_string(file_path.unwrap().path()).unwrap()).unwrap(); - - let start = std::time::Instant::now(); - // Benchmark the committer flow test. - commit(committer_input.0, OUTPUT_PATH.to_owned()).await; - let execution_time = std::time::Instant::now() - start; - - // Assert correctness of the output of the committer flow test. - let CommitterRegressionOutput { - contract_storage_root_hash, - compiled_class_root_hash, - storage: - StorageObject { - storage: Value::Object(storage_changes), - }, - } = serde_json::from_str(&std::fs::read_to_string(OUTPUT_PATH).unwrap()).unwrap() - else { - panic!("Expected the storage to be an object."); - }; - - assert_eq!(contract_storage_root_hash, expected_contract_states_root); - assert_eq!(compiled_class_root_hash, expected_contract_classes_root); - assert_eq!(storage_changes, *expected_facts); - - // Assert the execution time does not exceed the threshold. - assert!(execution_time.as_secs_f64() < MAX_TIME_FOR_COMMITTER_FLOW_BECHMARK_TEST); + // TODO(Aner, 23/07/24): multi-thread the test. + test_single_committer_flow( + &fs::read_to_string(file_path.unwrap().path()).unwrap(), + OUTPUT_PATH, + ) + .await; } - assert_eq!(file_counter, 100); }