Gather compilation statistics

.github/workflows/daily.yml

@@ -199,8 +199,7 @@ jobs:
         continue-on-error: true
 
       - name: Compare states
-        run: |
-          ./scripts/cmp_state_dumps.sh | tee output
+        run: python ./scripts/cmp_state_dumps.py | tee output
 
       - name: Upload Compare Results
         id: upload_compare_results

.github/workflows/starknet-blocks.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           repository: lambdaclass/starknet-replay
           path: starknet-replay
-          ref: 135ba7cd5b45fe137b11b7f75654dcd472363033
+          ref: 1b8e2e0be21a8df9f5f6b8f8514d1a40b456ef58
       # We need native to use the linux deps ci action
       - name: Checkout Native
         uses: actions/checkout@v4
@@ -43,7 +43,7 @@ jobs:
         with:
           repository: lambdaclass/sequencer
           path: sequencer
-          ref: b61262980394dab0e0a4c4cab85f12d31d0ce878
+          ref: 40331042c1149f5cb84b27f9dd8d47994a010bbe
 
       - name: Cache RPC Calls
         uses: actions/cache@v4.2.0
@@ -126,5 +126,4 @@ jobs:
         continue-on-error: true
 
       - name: Compare states
-        run: |
-          ./scripts/cmp_state_dumps.sh
+        run: python ./scripts/cmp_state_dumps.py

benches/compile_time.rs

@@ -15,7 +15,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
                 b.iter(|| {
                     let native_context = NativeContext::new();
                     native_context
-                        .compile(program, false, Some(Default::default()))
+                        .compile(program, false, Some(Default::default()), None)
                         .unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                 })
@@ -32,7 +32,7 @@ pub fn bench_compile_time(c: &mut Criterion) {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     native_context
-                        .compile(program, false, Some(Default::default()))
+                        .compile(program, false, Some(Default::default()), None)
                         .unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                 })
@@ -48,9 +48,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
                 b.iter(|| {
                     let native_context = NativeContext::new();
                     let module = native_context
-                        .compile(black_box(program), false, Some(Default::default()))
+                        .compile(black_box(program), false, Some(Default::default()), None)
                         .unwrap();
-                    let object = module_to_object(module.module(), OptLevel::None)
+                    let object = module_to_object(module.module(), OptLevel::None, None)
                         .expect("to compile correctly to a object file");
                     black_box(object)
                 })
@@ -67,9 +67,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let module = native_context
-                        .compile(black_box(program), false, Some(Default::default()))
+                        .compile(black_box(program), false, Some(Default::default()), None)
                         .unwrap();
-                    let object = module_to_object(module.module(), OptLevel::None)
+                    let object = module_to_object(module.module(), OptLevel::None, None)
                         .expect("to compile correctly to a object file");
                     black_box(object)
                 })
@@ -86,9 +86,9 @@ pub fn bench_compile_time(c: &mut Criterion) {
             c.bench_with_input(BenchmarkId::new(filename, 1), &program, |b, program| {
                 b.iter(|| {
                     let module = native_context
-                        .compile(black_box(program), false, Some(Default::default()))
+                        .compile(black_box(program), false, Some(Default::default()), None)
                         .unwrap();
-                    let object = module_to_object(module.module(), OptLevel::Aggressive)
+                    let object = module_to_object(module.module(), OptLevel::Aggressive, None)
                         .expect("to compile correctly to a object file");
                     black_box(object)
                 })

benches/libfuncs.rs

@@ -55,7 +55,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                     let native_context = NativeContext::new();
                     b.iter(|| {
                         let module = native_context
-                            .compile(program, false, Some(Default::default()))
+                            .compile(program, false, Some(Default::default()), None)
                             .unwrap();
                         // pass manager internally verifies the MLIR output is correct.
                         let native_executor =
@@ -77,7 +77,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                 |b, program| {
                     let native_context = NativeContext::new();
                     let module = native_context
-                        .compile(program, false, Some(Default::default()))
+                        .compile(program, false, Some(Default::default()), None)
                         .unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                     let native_executor =
@@ -108,7 +108,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                     let native_context = NativeContext::new();
                     b.iter(|| {
                         let module = native_context
-                            .compile(program, false, Some(Default::default()))
+                            .compile(program, false, Some(Default::default()), None)
                             .unwrap();
                         // pass manager internally verifies the MLIR output is correct.
                         let native_executor =
@@ -130,7 +130,7 @@ pub fn bench_libfuncs(c: &mut Criterion) {
                 |b, program| {
                     let native_context = NativeContext::new();
                     let module = native_context
-                        .compile(program, false, Some(Default::default()))
+                        .compile(program, false, Some(Default::default()), None)
                         .unwrap();
                     // pass manager internally verifies the MLIR output is correct.
                     let native_executor =

examples/easy_api.rs

@@ -16,7 +16,7 @@ fn main() {
 
     // Compile the sierra program into a MLIR module.
     let native_program = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     // The parameters of the entry point.

examples/erc20.rs

@@ -322,7 +322,7 @@ fn main() {
     let native_context = NativeContext::new();
 
     let native_program = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     let entry_point_fn =

examples/invoke.rs

@@ -21,7 +21,7 @@ fn main() {
     let native_context = NativeContext::new();
 
     let native_program = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     // Call the echo function from the contract using the generated wrapper.

examples/starknet.rs

@@ -456,7 +456,7 @@ fn main() {
     let native_context = NativeContext::new();
 
     let native_program = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     // Call the echo function from the contract using the generated wrapper.

scripts/cmp_state_dumps.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+#
+# usage: cmp-state-dumps [-h] [-d]
+# Compare all files in the state_dumps directory and outputs a summary
+# options:
+#   -h, --help    show this help message and exit
+#   -d, --delete  removes matching files
+#
+# Uses a pool of worker threads that compare each state dump.
+# possible improvements: use a pool of workers for file removing.
+
+import argparse
+import glob
+import re
+import multiprocessing as mp
+import os
+from collections import defaultdict
+
+POOL_SIZE = 16
+
+STATE_DUMPS_PATH = "state_dumps"
+VM_DIRECTORY = "vm"
+NATIVE_DIRECTORY = "native"
+
+LOG_PATH = "state_dumps/matching.log"
+
+
+def compare(vm_dump_path: str):
+    native_dump_path = re.sub(VM_DIRECTORY, NATIVE_DIRECTORY, vm_dump_path, count=1)
+
+    if not (m := re.findall(r"/(0x.*).json", vm_dump_path)):
+        raise Exception("bad path")
+    tx = m[0]
+
+    if not (m := re.findall(r"block(\d+)", vm_dump_path)):
+        raise Exception("bad path")
+    block = m[0]
+
+    try:
+        with open(native_dump_path) as f:
+            native_dump = f.read()
+        with open(vm_dump_path) as f:
+            vm_dump = f.read()
+    except:  # noqa: E722
+        return ("MISS", block, tx)
+
+    native_dump = re.sub(r".*reverted.*", "", native_dump, count=1)
+    vm_dump = re.sub(r".*reverted.*", "", vm_dump, count=1)
+
+    if native_dump == vm_dump:
+        return ("MATCH", block, tx, vm_dump_path, native_dump_path)
+    else:
+        return ("DIFF", block, tx)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="cmp-state-dumps",
+        description="Compare all files in the state_dumps directory and outputs a summary",
+    )
+    parser.add_argument(
+        "-d", "--delete", action="store_true", help="removes matching files"
+    )
+    config = parser.parse_args()
+
+    files = glob.glob(f"{STATE_DUMPS_PATH}/{VM_DIRECTORY}/*/*.json")
+    files.sort(key=os.path.getmtime)
+
+    print(f"Starting comparison with {POOL_SIZE} workers")
+
+    stats = defaultdict(int)
+    with mp.Pool(POOL_SIZE) as pool, open(LOG_PATH, mode="a") as log:
+        for status, *info in pool.imap(compare, files):
+            stats[status] += 1
+
+            if status != "MATCH":
+                (block, tx) = info
+                print(status, block, tx)
+
+            elif status == "MATCH" and config.delete:
+                (block, tx, vm_dump_path, native_dump_path) = info
+
+                log.write(f"{block} {tx}\n")
+                log.flush()
+                os.remove(native_dump_path)
+                os.remove(vm_dump_path)
+
+    print("Finished comparison")
+
+    print()
+    for key, count in stats.items():
+        print(key, count)
+
+    if stats["DIFF"] != 0 or stats["MISS"] != 0:
+        exit(1)
+    else:
+        exit(0)

src/bin/cairo-native-compile.rs

@@ -56,7 +56,7 @@ fn main() -> anyhow::Result<()> {
 
     // Compile the sierra program into a MLIR module.
     let native_module = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     let output_mlir = args
@@ -79,9 +79,10 @@ fn main() -> anyhow::Result<()> {
         })
     });
 
-    let object_data = module_to_object(native_module.module(), args.opt_level.into())
+    let object_data = module_to_object(native_module.module(), args.opt_level.into(), None)
         .context("Failed to convert module to object.")?;
-    object_to_shared_lib(&object_data, &output_lib).context("Failed to write shared library.")?;
+    object_to_shared_lib(&object_data, &output_lib, None)
+        .context("Failed to write shared library.")?;
 
     Ok(())
 }

src/bin/cairo-native-dump.rs

@@ -35,7 +35,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let program = load_program(Path::new(&args.input), args.starknet)?;
 
     // Compile the program.
-    let module = context.compile(&program, false, Some(Default::default()))?;
+    let module = context.compile(&program, false, Some(Default::default()), None)?;
 
     // Write the output.
     let output_str = module

src/bin/cairo-native-run.rs

@@ -92,7 +92,7 @@ fn main() -> anyhow::Result<()> {
 
     // Compile the sierra program into a MLIR module.
     let native_module = native_context
-        .compile(&sierra_program, false, Some(Default::default()))
+        .compile(&sierra_program, false, Some(Default::default()), None)
         .unwrap();
 
     let native_executor: Box<dyn Fn(_, _, _, &mut StubSyscallHandler) -> _> = match args.run_mode {

src/bin/cairo-native-stress/main.rs

@@ -280,7 +280,7 @@ where
     ) -> Arc<AotNativeExecutor> {
         let native_module = self
             .context
-            .compile(program, false, Some(Default::default()))
+            .compile(program, false, Some(Default::default()), None)
             .expect("failed to compile program");
 
         let registry = ProgramRegistry::new(program).expect("failed to get program registry");
@@ -291,15 +291,15 @@ where
             .expect("module should have gas metadata");
 
         let shared_library = {
-            let object_data = module_to_object(native_module.module(), opt_level)
+            let object_data = module_to_object(native_module.module(), opt_level, None)
                 .expect("failed to convert MLIR to object");
 
             let shared_library_dir = Path::new(AOT_CACHE_DIR);
             create_dir_all(shared_library_dir).expect("failed to create shared library directory");
             let shared_library_name = format!("lib{key}{SHARED_LIBRARY_EXT}");
             let shared_library_path = shared_library_dir.join(shared_library_name);
 
-            object_to_shared_lib(&object_data, &shared_library_path)
+            object_to_shared_lib(&object_data, &shared_library_path, None)
                 .expect("failed to link object into shared library");
 
             unsafe {

src/bin/scarb-native-dump.rs

@@ -41,6 +41,7 @@ fn main() -> anyhow::Result<()> {
                         &compiled.into_v1().unwrap().program,
                         false,
                         Some(Default::default()),
+                        None,
                     )
                     .unwrap();