New benchmark durations

.env

@@ -11,7 +11,7 @@ MEMORY_CIRCUIT_SIZE=17..28
 # Persistent-Leader specific
 PALADIN_SERIALIZER=POSTCARD # POSTCARD/CMOP (if not provided, uses value from `default` trait)
 PALADIN_RUNTIME=MEMORY # MEMORY/AMQP: -> DEFAULTS TO MEMORY
-PALADIN_AMQP_NUM_WORKERS=1 # ONLY WHEN RUNTIME IS "MEMORY"
+# PALADIN_AMQP_NUM_WORKERS=1 # ONLY WHEN RUNTIME IS "MEMORY"
 PSM_CIRCUIT_PERSISTENCE=NONE # NONE/DISK
 # SERVER_ADDR=0.0.0.0:8080 # Defaults to "0.0.0.0:8080"
 # PROOF_OUT_LOCAL_DIRPATH=proofs_out/ 

.github/workflows/deploy.yml

@@ -118,4 +118,4 @@ jobs:
         run: |-
           pwd && ls -lh
           cd ./deploy/helm
-          helm upgrade zero-bin ./zero-bin -f ./zero-bin/values.yaml --set hull.config.specific.version=sha-${GITHUB_SHA} -n zkevm --install
+          helm upgrade zero-bin ./zero-bin -f ./zero-bin/values.yaml --set hull.config.specific.version=sha-${GITHUB_SHA} -n zkevm --install

.gitignore

@@ -13,3 +13,7 @@ proofs/
 # Serialized generated prover & verifier state used by plonky2
 prover_state_*
 verifier_state_*
+
+# Coordinator output
+coordinator/benchmark_out/
+coordinator/proof_out/

Cargo.toml

@@ -43,3 +43,6 @@ opt-level = 3
 incremental = true
 lto = "fat"
 codegen-units = 1
+
+# [build]
+# rustflags = ["-C", "target-cpu=native"]

coordinator.Dockerfile

@@ -23,7 +23,6 @@ COPY leader/Cargo.toml ./leader/Cargo.toml
 COPY coordinator/Cargo.toml ./coordinator/Cargo.toml
 
 COPY ./rust-toolchain.toml ./
-
 RUN cargo build --release --bin coordinator 
 
 COPY coordinator ./coordinator
@@ -41,7 +40,7 @@ RUN \
   touch leader/src/main.rs && \
   touch coordinator/src/main.rs
 
-RUN cargo build --release --bin coordinator 
+RUN cargo build --release --bin coordinator
 
 FROM debian:bullseye-slim
 RUN apt-get update && apt-get install -y ca-certificates libjemalloc2

coordinator/README.md

@@ -119,6 +119,22 @@ The example below proves blocks [1,10] using the RPC function listed in ZeroBin,
 }
 ```
 
+```json
+{
+  "run_name": "run",
+  "start_block_number": 18,
+  "terminate_on": {
+    "EndBlock": {"block_number": 21}
+  },
+  "block_source": {
+    "ZeroBinRpc": {"rpc_url": "http://35.208.84.178:8545/"}
+  },
+  "benchmark_output": {
+    "LocalCsv": {"file_name": "test.csv"}
+  }
+}
+```
+
 An example not recording the proofs, and posting the results to a google cloud storage bucket.
 
 ```json

coordinator/src/benchmarking.rs

@@ -31,7 +31,11 @@ pub struct BenchmarkingStats {
     pub fetch_duration: Duration,
     /// The amount of time elapsed during the process of proving this block,
     /// stored as a [Duration]
-    pub proof_duration: Duration,
+    pub total_proof_duration: Duration,
+
+    pub prep_duration: Option<Duration>,
+    pub txproof_duration: Option<Duration>,
+    pub agg_duration: Option<Duration>,
     /// The start time of the proof.  [BenchmarkingStats::proof_duration] is a
     /// more reliable value to use for the proof duration.  Timestamps measured
     /// in UTC.
@@ -61,7 +65,7 @@ impl BenchmarkingStats {
     /// Returns a header row
     pub fn header_row() -> String {
         String::from(
-            "block_number, number_txs, cumulative_number_txs, fetch_duration, proof_duration, start_time, end_time, overall_elapsed_time, proof_out_duration, gas_used, gas_used_per_tx, cumulative_gas_used, difficulty",
+            "block_number, number_txs, cumulative_number_txs, fetch_duration, total_proof_duration, prep_duration, txproof_duration, agg_duration, start_time, end_time, overall_elapsed_time, proof_out_duration, gas_used, cumulative_gas_used, difficulty, gas_used_per_tx",
         )
     }
 
@@ -98,24 +102,27 @@ impl BenchmarkingStats {
     #[allow(clippy::format_in_format_args)]
     pub fn as_csv_row(&self) -> String {
         format!(
-            "{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, \"{}\", {}, {}",
+            "{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, \"{}\"",
             self.block_number,
             self.n_txs,
             Self::unwrap_to_string(self.cumulative_n_txs),
             self.fetch_duration.as_secs_f64(),
-            self.proof_duration.as_secs_f64(),
+            self.total_proof_duration.as_secs_f64(),
+            Self::unwrap_duration_to_string(self.prep_duration),
+            Self::unwrap_duration_to_string(self.txproof_duration),
+            Self::unwrap_duration_to_string(self.agg_duration),
             self.start_time.format("%d-%m-%Y %H:%M:%S"),
             self.end_time.format("%d-%m-%Y %H:%M:%S"),
             Self::unwrap_to_string(self.overall_elapsed_seconds),
             Self::unwrap_duration_to_string(self.proof_out_duration),
             self.gas_used,
+            Self::unwrap_to_string(self.cumulative_gas_used),
+            self.difficulty,
             self.gas_used_per_tx
                 .iter()
                 .map(|gas| gas.to_string())
                 .collect::<Vec<String>>()
                 .join(";"),
-            Self::unwrap_to_string(self.cumulative_gas_used),
-            self.difficulty,
         )
     }
 }

coordinator/src/main.rs

@@ -284,7 +284,7 @@ fn build_paladin_config_from_env() -> Config {
                 "Number of workers not specified for InMemory runtime, using default: {}",
                 DFLT_NUM_WORKERS
             );
-            Some(DFLT_NUM_WORKERS)
+            None //Some(DFLT_NUM_WORKERS)
         }
         (paladin::config::Runtime::InMemory, Err(env::VarError::NotUnicode(os_str))) => {
             info!("Non-Unicode input for number of workers: {:?}", os_str);

coordinator/src/manyprover.rs

@@ -308,20 +308,23 @@ impl ManyProver {
             info!("Starting to prove block {}", block_num);
             let proof_start_instance = Instant::now();
             let proof_start_stamp: DateTime<Utc> = SystemTime::now().into();
-            let proof = match prover_input.prove(runtime.as_ref(), None, true).await {
-                Ok(proof) => proof,
+            let benchmarked_proof = match prover_input
+                .prove_and_benchmark(runtime.as_ref(), None, true)
+                .await
+            {
+                Ok(benchmarked_proof) => benchmarked_proof,
                 Err(err) => {
                     error!("Failed to generate block {}'s proof: {}", block_num, err);
                     return Err(ManyProverError::Proof(err));
                 }
             };
 
-            let proof_duration = proof_start_instance.elapsed();
+            let total_proof_duration = proof_start_instance.elapsed();
             let proof_end_stamp: DateTime<Utc> = SystemTime::now().into();
             info!(
                 "Proved block {} in {} seconds",
                 block_num,
-                proof_duration.as_secs_f64()
+                total_proof_duration.as_secs_f64()
             );
 
             // Create the benchmarking statistics and return both.
@@ -331,7 +334,10 @@ impl ManyProver {
                     n_txs: n_txs as u64,
                     cumulative_n_txs: None,
                     fetch_duration,
-                    proof_duration,
+                    total_proof_duration,
+                    prep_duration: benchmarked_proof.prep_dur,
+                    txproof_duration: benchmarked_proof.proof_dur,
+                    agg_duration: benchmarked_proof.agg_dur,
                     start_time: proof_start_stamp,
                     end_time: proof_end_stamp,
                     overall_elapsed_seconds: None,
@@ -341,7 +347,7 @@ impl ManyProver {
                     cumulative_gas_used: None,
                     difficulty,
                 },
-                proof,
+                proof: benchmarked_proof.proof,
             })
         }
 
@@ -762,7 +768,10 @@ impl ManyProver {
             let proof_start_instance = Instant::now();
             // The stamp will signify the starting process of this proof.
             let proof_start_stamp: DateTime<Utc> = SystemTime::now().into();
-            let proof = match prover_input.prove(self.runtime.as_ref(), prev, true).await {
+            let benchmarked_proof = match prover_input
+                .prove_and_benchmark(self.runtime.as_ref(), prev, true)
+                .await
+            {
                 Ok(proof) => proof,
                 Err(err) => {
                     error!(
@@ -820,7 +829,7 @@ impl ManyProver {
             let proof_out_time: Option<std::time::Duration> = match &self.proof_out {
                 Some(proof_out) => {
                     let proof_out_start = Instant::now();
-                    match proof_out.write(&proof) {
+                    match proof_out.write(&benchmarked_proof.proof) {
                         Ok(_) => {
                             info!("Successfully wrote proof");
                             Some(proof_out_start.elapsed())
@@ -837,7 +846,7 @@ impl ManyProver {
             // If we need to keep the proof, save it in prev, otherwise do not.
             prev = match self.input_request.forward_prev {
                 Some(false) | None => None,
-                Some(true) => Some(proof.intern),
+                Some(true) => Some(benchmarked_proof.proof.intern),
             };
 
             //------------------------------------------------------------------------
@@ -852,7 +861,10 @@ impl ManyProver {
                     n_txs,
                     cumulative_n_txs: Some(cumulative_n_txs),
                     fetch_duration,
-                    proof_duration,
+                    total_proof_duration: proof_duration,
+                    prep_duration: benchmarked_proof.prep_dur,
+                    txproof_duration: benchmarked_proof.proof_dur,
+                    agg_duration: benchmarked_proof.agg_dur,
                     start_time: proof_start_stamp,
                     end_time: proof_end_stamp,
                     overall_elapsed_seconds: Some(

leader.Dockerfile

@@ -36,7 +36,7 @@ RUN \
   touch prover/src/lib.rs && \
   touch leader/src/main.rs
 
-RUN cargo build --release --bin leader 
+RUN cargo build --release --bin leader
 
 FROM debian:bullseye-slim
 RUN apt-get update && apt-get install -y ca-certificates libjemalloc2

prover/src/lib.rs

@@ -1,3 +1,5 @@
+use std::time::{Duration, Instant};
+
 use anyhow::Result;
 use ethereum_types::U256;
 #[cfg(feature = "test_only")]
@@ -14,7 +16,6 @@ use trace_decoder::{
     trace_protocol::BlockTrace,
     types::{CodeHash, OtherBlockData},
 };
-#[cfg(feature = "test_only")]
 use tracing::info;
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -26,11 +27,101 @@ fn resolve_code_hash_fn(_: &CodeHash) -> Vec<u8> {
     todo!()
 }
 
+pub struct BenchmarkedGeneratedBlockProof {
+    pub proof: GeneratedBlockProof,
+    pub prep_dur: Option<Duration>,
+    pub proof_dur: Option<Duration>,
+    pub agg_dur: Option<Duration>,
+}
+
+impl From<BenchmarkedGeneratedBlockProof> for GeneratedBlockProof {
+    fn from(value: BenchmarkedGeneratedBlockProof) -> Self {
+        value.proof
+    }
+}
+
 impl ProverInput {
     pub fn get_block_number(&self) -> U256 {
         self.other_data.b_data.b_meta.block_number
     }
 
+    #[cfg(not(feature = "test_only"))]
+    pub async fn prove_and_benchmark(
+        self,
+        runtime: &Runtime,
+        previous: Option<PlonkyProofIntern>,
+        save_inputs_on_error: bool,
+    ) -> Result<BenchmarkedGeneratedBlockProof> {
+        let prep_start = Instant::now();
+
+        let block_number = self.get_block_number();
+        let other_data = self.other_data;
+        let txs = self.block_trace.into_txn_proof_gen_ir(
+            &ProcessingMeta::new(resolve_code_hash_fn),
+            other_data.clone(),
+        )?;
+
+        let prep_dur = prep_start.elapsed();
+
+        info!(
+            "Completed pre-proof work for block {} in {} secs",
+            block_number,
+            prep_dur.as_secs_f64()
+        );
+
+        let proof_start = Instant::now();
+        let agg_proof = IndexedStream::from(txs)
+            .map(&TxProof {
+                save_inputs_on_error,
+            })
+            .fold(&ops::AggProof {
+                save_inputs_on_error,
+            })
+            .run(runtime)
+            .await?;
+        let proof_dur = proof_start.elapsed();
+
+        info!(
+            "Completed tx proofs for block {} in {} secs",
+            block_number,
+            proof_dur.as_secs_f64()
+        );
+
+        if let proof_gen::proof_types::AggregatableProof::Agg(proof) = agg_proof {
+            let agg_start = Instant::now();
+            let prev = previous.map(|p| GeneratedBlockProof {
+                b_height: block_number.as_u64() - 1,
+                intern: p,
+            });
+
+            let block_proof = paladin::directive::Literal(proof)
+                .map(&ops::BlockProof {
+                    prev,
+                    save_inputs_on_error,
+                })
+                .run(runtime)
+                .await?;
+
+            let agg_dur = agg_start.elapsed();
+
+            info!(
+                "Completed tx proof agg for block {} in {} secs",
+                block_number,
+                agg_dur.as_secs_f64()
+            );
+
+            // Return the block proof
+            Ok(BenchmarkedGeneratedBlockProof {
+                proof: block_proof.0,
+                proof_dur: Some(proof_dur),
+                prep_dur: Some(prep_dur),
+                agg_dur: Some(agg_dur),
+            })
+        } else {
+            anyhow::bail!("AggProof is is not GeneratedAggProof")
+        }
+    }
+
     /// Evaluates a singular block
     #[cfg(not(feature = "test_only"))]
     pub async fn prove(