feat: implement windowed porep

Author: dignifiedquire

.circleci/config.yml

@@ -181,7 +181,7 @@ jobs:
       - run:
           name: Ensure existence of Groth parameters and keys on remote host
           command: |
-            ./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024))"
+            ./fil-proofs-tooling/scripts/paramcache-remote.sh "${CIRCLE_BRANCH}" "${BENCHMARK_SERVER_SSH_USERNAME}@${BENCHMARK_SERVER_IP_ADDR}" "-z=$((1024*1024*1024)):$(((128*1024*1024)/32))"
           no_output_timeout: 60m
       - run:
           name: Run hash-constraints benchmarks on remote host
@@ -365,7 +365,8 @@ workflows:
             - cargo_fetch
           filters:
             branches:
-              only: master
+              only:
+                - master
       - build_linux_release:
           requires:
             - cargo_fetch

fil-proofs-tooling/src/bin/benchy/election_post.rs

@@ -52,8 +52,12 @@ impl Report {
     }
 }
 
-pub fn run(sector_size: usize) -> Result<(), failure::Error> {
-    info!("Benchy Election PoSt: sector-size={}", sector_size);
+pub fn run(sector_size: usize, window_size_nodes: usize) -> Result<(), failure::Error> {
+    info!(
+        "Benchy Election PoSt: sector-size={} - window_size={}",
+        sector_size,
+        window_size_nodes * 32
+    );
 
     let sector_size_unpadded_bytes_ammount =
         UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size as u64));
@@ -95,7 +99,11 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
     let piece_infos = vec![piece_info];
 
     // Replicate the staged sector, write the replica file to `sealed_path`.
-    let porep_config = PoRepConfig(SectorSize(sector_size as u64), N_PARTITIONS);
+    let porep_config = PoRepConfig {
+        sector_size: SectorSize(sector_size as u64),
+        partitions: N_PARTITIONS,
+        window_size_nodes,
+    };
     let cache_dir = tempfile::tempdir().unwrap();
     let sector_id = SectorId::from(SECTOR_ID);
     let ticket = [0u8; 32];
@@ -137,7 +145,10 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
     );
 
     // Measure PoSt generation and verification.
-    let post_config = PoStConfig(SectorSize(sector_size as u64));
+    let post_config = PoStConfig {
+        sector_size: SectorSize(sector_size as u64),
+        window_size_nodes,
+    };
 
     let challenge_count = 1u64;
 

fil-proofs-tooling/src/bin/benchy/main.rs

@@ -20,12 +20,27 @@ fn main() {
                         .takes_value(true),
                 )
                 .arg(
-                    Arg::with_name("challenges")
-                        .long("challenges")
-                        .help("How many challenges to execute")
+                    Arg::with_name("window-size")
+                        .long("window-size")
+                        .help("The window size in bytes")
+                        .default_value("4096")
+                        .takes_value(true),
+                )
+                .arg(
+                    Arg::with_name("wrapper-challenges")
+                        .long("wrapper-challenges")
+                        .help("How many wrapper challenges to execute")
                         .default_value("1")
                         .takes_value(true),
                 )
+                .arg(
+                    Arg::with_name("window-challenges")
+                        .long("window-challenges")
+                        .help("How many window challenges to execute")
+                        .default_value("1")
+                        .takes_value(true),
+                )
+
                 .arg(
                     Arg::with_name("hasher")
                         .long("hasher")
@@ -97,6 +112,13 @@ fn main() {
                 .required(true)
                 .help("The data size in KiB")
                 .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("window-size")
+                .long("window-size")
+                .help("The window size in bytes")
+                .default_value("4096")
+                .takes_value(true),
         );
 
     let hash_cmd = SubCommand::with_name("hash-constraints")
@@ -114,10 +136,16 @@ fn main() {
             Ok(())
                 .and_then(|_| {
                     let layers = value_t!(m, "layers", usize)?;
+                    let window_size_bytes = value_t!(m, "window-size", usize)
+                        .expect("could not convert `window-size` CLI argument to `usize`");
+                    let window_size_nodes = window_size_bytes / 32;
+
                     stacked::run(stacked::RunOpts {
                         bench: m.is_present("bench"),
                         bench_only: m.is_present("bench-only"),
-                        challenges: value_t!(m, "challenges", usize)?,
+                        window_size_nodes,
+                        window_challenges: value_t!(m, "window-challenges", usize)?,
+                        wrapper_challenges: value_t!(m, "wrapper-challenges", usize)?,
                         circuit: m.is_present("circuit"),
                         dump: m.is_present("dump"),
                         extract: m.is_present("extract"),
@@ -136,7 +164,10 @@ fn main() {
             let sector_size_kibs = value_t!(m, "size", usize)
                 .expect("could not convert `size` CLI argument to `usize`");
             let sector_size = sector_size_kibs * 1024;
-            election_post::run(sector_size).expect("election-post failed");
+            let window_size_bytes = value_t!(m, "window-size", usize)
+                .expect("could not convert `window-size` CLI argument to `usize`");
+            let window_size_nodes = window_size_bytes / 32;
+            election_post::run(sector_size, window_size_nodes).expect("election-post failed");
         }
         ("hash-constraints", Some(_m)) => {
             hash_fns::run().expect("hash-constraints failed");

fil-proofs-tooling/src/bin/benchy/stacked.rs

@@ -21,8 +21,7 @@ use storage_proofs::hasher::{Blake2sHasher, Domain, Hasher, PedersenHasher, Sha2
 use storage_proofs::porep::PoRep;
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::stacked::{
-    self, CacheKey, ChallengeRequirements, LayerChallenges, StackedDrg, TemporaryAuxCache,
-    EXP_DEGREE,
+    self, CacheKey, ChallengeRequirements, StackedConfig, StackedDrg, TemporaryAuxCache, EXP_DEGREE,
 };
 use tempfile::TempDir;
 
@@ -46,7 +45,7 @@ fn file_backed_mmap_from_zeroes(n: usize, use_tmp: bool) -> Result<MmapMut, fail
 }
 
 fn dump_proof_bytes<H: Hasher>(
-    all_partition_proofs: &[Vec<stacked::Proof<H, Sha256Hasher>>],
+    all_partition_proofs: &[stacked::Proof<H, Sha256Hasher>],
 ) -> Result<(), failure::Error> {
     let file = OpenOptions::new()
         .write(true)
@@ -62,8 +61,9 @@ fn dump_proof_bytes<H: Hasher>(
 #[derive(Clone, Debug)]
 struct Params {
     samples: usize,
+    window_size_nodes: usize,
     data_size: usize,
-    layer_challenges: LayerChallenges,
+    config: StackedConfig,
     partitions: usize,
     circuit: bool,
     groth: bool,
@@ -82,10 +82,10 @@ impl From<Params> for Inputs {
             partitions: p.partitions,
             hasher: p.hasher.clone(),
             samples: p.samples,
-            layers: p.layer_challenges.layers(),
-            partition_challenges: p.layer_challenges.challenges_count_all(),
-            total_challenges: p.layer_challenges.challenges_count_all() * p.partitions,
-            layer_challenges: p.layer_challenges,
+            layers: p.config.layers(),
+            partition_challenges: p.config.window_challenges.challenges_count_all(),
+            total_challenges: p.config.window_challenges.challenges_count_all() * p.partitions,
+            config: p.config,
         }
     }
 }
@@ -110,7 +110,7 @@ where
         let Params {
             samples,
             data_size,
-            layer_challenges,
+            config,
             partitions,
             circuit,
             groth,
@@ -119,12 +119,13 @@ where
             use_tmp,
             dump_proofs,
             bench_only,
+            window_size_nodes,
             ..
         } = &params;
 
         // MT for original data is always named tree-d, and it will be
         // referenced later in the process as such.
-        let config = StoreConfig::new(
+        let store_config = StoreConfig::new(
             cache_dir.path(),
             CacheKey::CommDTree.to_string(),
             DEFAULT_CACHED_ABOVE_BASE_LAYER,
@@ -142,7 +143,8 @@ where
             degree: BASE_DEGREE,
             expansion_degree: EXP_DEGREE,
             seed: new_seed(),
-            layer_challenges: layer_challenges.clone(),
+            config: config.clone(),
+            window_size_nodes: *window_size_nodes,
         };
 
         let pp = StackedDrg::<H, Sha256Hasher>::setup(&sp)?;
@@ -163,7 +165,7 @@ where
                     &replica_id,
                     &mut data,
                     None,
-                    Some(config.clone()),
+                    Some(store_config.clone()),
                 )?;
 
                 let pb = stacked::PublicInputs::<H::Domain, <Sha256Hasher as Hasher>::Domain> {
@@ -292,7 +294,7 @@ where
                         &pp,
                         &replica_id,
                         &data,
-                        Some(config.clone()),
+                        Some(store_config.clone()),
                     )
                     .map_err(|err| err.into())
                 })?;
@@ -348,6 +350,7 @@ fn do_circuit_work<H: 'static + Hasher>(
     };
 
     if *bench || *circuit {
+        info!("Generating blank circuit");
         let mut cs = MetricCS::<Bls12>::new();
         <StackedCompound as CompoundProof<_, StackedDrg<H, Sha256Hasher>, _>>::blank_circuit(&pp)
             .synthesize(&mut cs)?;
@@ -357,6 +360,7 @@ fn do_circuit_work<H: 'static + Hasher>(
     }
 
     if *groth {
+        info!("Generating Groth Proof");
         let pub_inputs = pub_in.expect("missing public inputs");
         let priv_inputs = priv_in.expect("missing private inputs");
 
@@ -438,7 +442,7 @@ struct Inputs {
     layers: usize,
     partition_challenges: usize,
     total_challenges: usize,
-    layer_challenges: LayerChallenges,
+    config: StackedConfig,
 }
 
 #[derive(Serialize, Default)]
@@ -484,7 +488,9 @@ impl Report {
 pub struct RunOpts {
     pub bench: bool,
     pub bench_only: bool,
-    pub challenges: usize,
+    pub window_size_nodes: usize,
+    pub window_challenges: usize,
+    pub wrapper_challenges: usize,
     pub circuit: bool,
     pub dump: bool,
     pub extract: bool,
@@ -498,10 +504,10 @@ pub struct RunOpts {
 }
 
 pub fn run(opts: RunOpts) -> Result<(), failure::Error> {
-    let layer_challenges = LayerChallenges::new(opts.layers, opts.challenges);
+    let config = StackedConfig::new(opts.layers, opts.window_challenges, opts.wrapper_challenges);
 
     let params = Params {
-        layer_challenges,
+        config,
         data_size: opts.size * 1024,
         partitions: opts.partitions,
         use_tmp: !opts.no_tmp,
@@ -512,6 +518,7 @@ pub fn run(opts: RunOpts) -> Result<(), failure::Error> {
         circuit: opts.circuit,
         extract: opts.extract,
         hasher: opts.hasher,
+        window_size_nodes: opts.window_size_nodes,
         samples: 5,
     };
 

fil-proofs-tooling/src/bin/micro.rs

@@ -66,6 +66,7 @@ fn parse_criterion_out(s: impl AsRef<str>) -> Result<Vec<CriterionResult>, failu
     let median_re = make_detail_re("median");
     let med_abs_dev_re = make_detail_re(r"med\. abs\. dev\.");
 
+    #[allow(clippy::type_complexity)]
     let mut current: Option<(
         String,
         Option<u32>,

filecoin-proofs/examples/stacked.rs

@@ -34,8 +34,7 @@ use storage_proofs::hasher::{Blake2sHasher, Domain, Hasher, PedersenHasher, Sha2
 use storage_proofs::porep::PoRep;
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::stacked::{
-    self, CacheKey, ChallengeRequirements, LayerChallenges, StackedDrg, TemporaryAuxCache,
-    EXP_DEGREE,
+    self, CacheKey, ChallengeRequirements, StackedConfig, StackedDrg, TemporaryAuxCache, EXP_DEGREE,
 };
 
 // We can only one of the profilers at a time, either CPU (`profile`)
@@ -129,7 +128,7 @@ pub fn file_backed_mmap_from(data: &[u8]) -> MmapMut {
     unsafe { MmapOptions::new().map_mut(&tmpfile).unwrap() }
 }
 
-fn dump_proof_bytes<H: Hasher, G: Hasher>(all_partition_proofs: &[Vec<stacked::Proof<H, G>>]) {
+fn dump_proof_bytes<H: Hasher, G: Hasher>(all_partition_proofs: &[stacked::Proof<H, G>]) {
     let file = OpenOptions::new()
         .write(true)
         .create(true)
@@ -144,7 +143,7 @@ fn dump_proof_bytes<H: Hasher, G: Hasher>(all_partition_proofs: &[Vec<stacked::P
 
 fn do_the_work<H: 'static>(
     data_size: usize,
-    layer_challenges: LayerChallenges,
+    config: StackedConfig,
     partitions: usize,
     circuit: bool,
     groth: bool,
@@ -164,8 +163,8 @@ fn do_the_work<H: 'static>(
     info!("data size: {}", prettyb(data_size));
     info!("m: {}", m);
     info!("expansion_degree: {}", expansion_degree);
-    info!("layer_challenges: {:?}", layer_challenges);
-    info!("layers: {}", layer_challenges.layers());
+    info!("config: {:?}", config);
+    info!("layers: {}", config.layers());
     info!("partitions: {}", partitions);
     info!("circuit: {:?}", circuit);
     info!("groth: {:?}", groth);
@@ -179,7 +178,8 @@ fn do_the_work<H: 'static>(
         degree: m,
         expansion_degree,
         seed: new_seed(),
-        layer_challenges: layer_challenges.clone(),
+        config: config.clone(),
+        window_size_nodes: nodes / 4,
     };
 
     info!("running setup");
@@ -494,7 +494,8 @@ fn main() {
         .get_matches();
 
     let data_size = value_t!(matches, "size", usize).unwrap() * 1024;
-    let challenge_count = value_t!(matches, "challenges", usize).unwrap();
+    let window_challenge_count = value_t!(matches, "window-challenges", usize).unwrap();
+    let wrapper_challenge_count = value_t!(matches, "wrapper-challenges", usize).unwrap();
     let hasher = value_t!(matches, "hasher", String).unwrap();
     let layers = value_t!(matches, "layers", usize).unwrap();
     let partitions = value_t!(matches, "partitions", usize).unwrap();
@@ -506,14 +507,14 @@ fn main() {
     let circuit = matches.is_present("circuit");
     let extract = matches.is_present("extract");
 
-    let challenges = LayerChallenges::new(layers, challenge_count);
+    let config = StackedConfig::new(layers, window_challenge_count, wrapper_challenge_count);
 
     info!("hasher: {}", hasher);
     match hasher.as_ref() {
         "pedersen" => {
             do_the_work::<PedersenHasher>(
                 data_size,
-                challenges,
+                config,
                 partitions,
                 circuit,
                 groth,
@@ -527,7 +528,7 @@ fn main() {
         "sha256" => {
             do_the_work::<Sha256Hasher>(
                 data_size,
-                challenges,
+                config,
                 partitions,
                 circuit,
                 groth,
@@ -541,7 +542,7 @@ fn main() {
         "blake2s" => {
             do_the_work::<Blake2sHasher>(
                 data_size,
-                challenges,
+                config,
                 partitions,
                 circuit,
                 groth,