feat(filecoin-proofs): api for deals on chain and interactive po… (#912)

feat(filecoin-proofs): api for deals on chain and interactive porep

Author: dignifiedquire

fil-proofs-tooling/src/bin/benchy/rational_post.rs

@@ -1,16 +1,16 @@
 use std::collections::BTreeMap;
-use std::io::stdout;
+use std::io::{stdout, Seek, SeekFrom, Write};
 
 use fil_proofs_tooling::{measure, Metadata};
-use filecoin_proofs::fr32::write_padded;
-use filecoin_proofs::pieces::get_aligned_source;
 use filecoin_proofs::types::{
     PaddedBytesAmount, PoRepConfig, PoRepProofPartitions, PoStConfig, SectorSize,
     UnpaddedBytesAmount,
 };
-use filecoin_proofs::{generate_post, seal, verify_post, PrivateReplicaInfo, PublicReplicaInfo};
+use filecoin_proofs::{
+    add_piece, generate_piece_commitment, generate_post, seal_commit, seal_pre_commit, verify_post,
+    PrivateReplicaInfo, PublicReplicaInfo,
+};
 use log::info;
-use rand::random;
 use storage_proofs::sector::SectorId;
 use tempfile::NamedTempFile;
 
@@ -58,8 +58,7 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
         UnpaddedBytesAmount::from(PaddedBytesAmount(sector_size as u64));
 
     // Create files for the staged and sealed sectors.
-    let mut staged_file =
-        NamedTempFile::new().expect("could not create temp file for staged sector");
+    let staged_file = NamedTempFile::new().expect("could not create temp file for staged sector");
 
     let sealed_file = NamedTempFile::new().expect("could not create temp file for sealed sector");
 
@@ -71,47 +70,68 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
 
     // Generate the data from which we will create a replica, we will then prove the continued
     // storage of that replica using the PoSt.
-    let data: Vec<u8> = (0..sector_size).map(|_| random()).collect();
+    let piece_bytes: Vec<u8> = (0..sector_size).map(|_| rand::random::<u8>()).collect();
+
+    let mut piece_file = NamedTempFile::new()?;
+    piece_file.write_all(&piece_bytes)?;
+    piece_file.as_file_mut().sync_all()?;
+    piece_file.as_file_mut().seek(SeekFrom::Start(0))?;
+
+    let piece_info =
+        generate_piece_commitment(piece_file.as_file_mut(), sector_size_unpadded_bytes_ammount)?;
+    piece_file.as_file_mut().seek(SeekFrom::Start(0))?;
 
-    // Write the aligned data to the staged sector file.
-    let (_, mut aligned_data) =
-        get_aligned_source(&data[..], &[], sector_size_unpadded_bytes_ammount);
+    let mut staged_sector_file = NamedTempFile::new()?;
+    add_piece(
+        &mut piece_file,
+        &mut staged_sector_file,
+        sector_size_unpadded_bytes_ammount,
+        &[],
+    )?;
 
-    write_padded(&mut aligned_data, &mut staged_file)
-        .expect("failed to write padded data to staged sector file");
+    let piece_infos = vec![piece_info];
 
     // Replicate the staged sector, write the replica file to `sealed_path`.
     let porep_config = PoRepConfig(SectorSize(sector_size as u64), N_PARTITIONS);
     let cache_dir = tempfile::tempdir().unwrap();
     let sector_id = SectorId::from(SECTOR_ID);
     let ticket = [0u8; 32];
 
-    let seal_output = seal(
+    let seal_pre_commit_output = seal_pre_commit(
         porep_config,
         cache_dir.path(),
         staged_file.path(),
         sealed_file.path(),
         PROVER_ID,
         sector_id,
         ticket,
-        &[sector_size_unpadded_bytes_ammount],
-    )
-    .expect("failed to seal");
+        &piece_infos,
+    )?;
+
+    let seed = [0u8; 32];
+    let comm_r = seal_pre_commit_output.comm_r;
+    let p_aux = seal_pre_commit_output.p_aux.clone();
+
+    let _seal_commit_output = seal_commit(
+        porep_config,
+        cache_dir.path(),
+        PROVER_ID,
+        sector_id,
+        ticket,
+        seed,
+        seal_pre_commit_output,
+        &piece_infos,
+    )?;
 
     // Store the replica's private and publicly facing info for proving and verifying respectively.
     let mut pub_replica_info: BTreeMap<SectorId, PublicReplicaInfo> = BTreeMap::new();
     let mut priv_replica_info: BTreeMap<SectorId, PrivateReplicaInfo> = BTreeMap::new();
 
-    pub_replica_info.insert(sector_id, PublicReplicaInfo::new(seal_output.comm_r));
+    pub_replica_info.insert(sector_id, PublicReplicaInfo::new(comm_r));
 
     priv_replica_info.insert(
         sector_id,
-        PrivateReplicaInfo::new(
-            sealed_path_string,
-            seal_output.comm_r,
-            seal_output.p_aux,
-            cache_dir.into_path(),
-        ),
+        PrivateReplicaInfo::new(sealed_path_string, comm_r, p_aux, cache_dir.into_path()),
     );
 
     // Measure PoSt generation and verification.

fil-proofs-tooling/src/bin/benchy/stacked.rs

@@ -44,7 +44,7 @@ fn file_backed_mmap_from_zeroes(n: usize, use_tmp: bool) -> Result<MmapMut, fail
 }
 
 fn dump_proof_bytes<H: Hasher>(
-    all_partition_proofs: &[Vec<stacked::Proof<H>>],
+    all_partition_proofs: &[Vec<stacked::Proof<H, Blake2sHasher>>],
 ) -> Result<(), failure::Error> {
     let file = OpenOptions::new()
         .write(true)
@@ -132,24 +132,25 @@ where
             layer_challenges: layer_challenges.clone(),
         };
 
-        let pp = StackedDrg::<H>::setup(&sp)?;
+        let pp = StackedDrg::<H, Blake2sHasher>::setup(&sp)?;
 
         let (pub_in, priv_in, d) = if *bench_only {
             (None, None, None)
         } else {
             let mut data = file_backed_mmap_from_zeroes(nodes, *use_tmp)?;
+            let seed = rng.gen();
 
             let FuncMeasurement {
                 cpu_time: replication_cpu_time,
                 wall_time: replication_wall_time,
                 return_value: (pub_inputs, priv_inputs),
             } = measure(|| {
                 let (tau, (p_aux, t_aux)) =
-                    StackedDrg::<H>::replicate(&pp, &replica_id, &mut data, None)?;
+                    StackedDrg::<H, Blake2sHasher>::replicate(&pp, &replica_id, &mut data, None)?;
 
-                let pb = stacked::PublicInputs::<H::Domain> {
+                let pb = stacked::PublicInputs::<H::Domain, <Blake2sHasher as Hasher>::Domain> {
                     replica_id,
-                    seed: None,
+                    seed,
                     tau: Some(tau.clone()),
                     k: Some(0),
                 };
@@ -188,8 +189,13 @@ where
                 wall_time: vanilla_proving_wall_time,
                 return_value: all_partition_proofs,
             } = measure(|| {
-                StackedDrg::<H>::prove_all_partitions(&pp, &pub_inputs, &priv_inputs, *partitions)
-                    .map_err(|err| err.into())
+                StackedDrg::<H, Blake2sHasher>::prove_all_partitions(
+                    &pp,
+                    &pub_inputs,
+                    &priv_inputs,
+                    *partitions,
+                )
+                .map_err(|err| err.into())
             })?;
 
             report.outputs.vanilla_proving_wall_time_us =
@@ -212,7 +218,7 @@ where
 
             for _ in 0..*samples {
                 let m = measure(|| {
-                    let verified = StackedDrg::<H>::verify_all_partitions(
+                    let verified = StackedDrg::<H, Blake2sHasher>::verify_all_partitions(
                         &pp,
                         &pub_inputs,
                         &all_partition_proofs,
@@ -259,7 +265,8 @@ where
         if let Some(data) = d {
             if *extract {
                 let m = measure(|| {
-                    StackedDrg::<H>::extract_all(&pp, &replica_id, &data).map_err(|err| err.into())
+                    StackedDrg::<H, Blake2sHasher>::extract_all(&pp, &replica_id, &data)
+                        .map_err(|err| err.into())
                 })?;
 
                 assert_ne!(&(*data), m.return_value.as_slice());
@@ -289,9 +296,9 @@ struct CircuitWorkMeasurement {
 }
 
 fn do_circuit_work<H: 'static + Hasher>(
-    pp: &<StackedDrg<H> as ProofScheme>::PublicParams,
-    pub_in: Option<<StackedDrg<H> as ProofScheme>::PublicInputs>,
-    priv_in: Option<<StackedDrg<H> as ProofScheme>::PrivateInputs>,
+    pp: &<StackedDrg<H, Blake2sHasher> as ProofScheme>::PublicParams,
+    pub_in: Option<<StackedDrg<H, Blake2sHasher> as ProofScheme>::PublicInputs>,
+    priv_in: Option<<StackedDrg<H, Blake2sHasher> as ProofScheme>::PrivateInputs>,
     params: &Params,
     report: &mut Report,
 ) -> Result<CircuitWorkMeasurement, failure::Error> {
@@ -315,7 +322,10 @@ fn do_circuit_work<H: 'static + Hasher>(
 
     if *bench || *circuit {
         let mut cs = MetricCS::<Bls12>::new();
-        StackedCompound::blank_circuit(&pp, &JJ_PARAMS).synthesize(&mut cs)?;
+        <StackedCompound as CompoundProof<_, StackedDrg<H, Blake2sHasher>, _>>::blank_circuit(
+            &pp, &JJ_PARAMS,
+        )
+        .synthesize(&mut cs)?;
 
         report.outputs.circuit_num_inputs = Some(cs.num_inputs() as u64);
         report.outputs.circuit_num_constraints = Some(cs.num_constraints() as u64);
@@ -332,7 +342,10 @@ fn do_circuit_work<H: 'static + Hasher>(
         // We should also allow the serialized vanilla proofs to be passed (as a file) to the example
         // and skip replication/vanilla-proving entirely.
         let gparams =
-            StackedCompound::groth_params(&compound_public_params.vanilla_params, &JJ_PARAMS)?;
+            <StackedCompound as CompoundProof<_, StackedDrg<H, Blake2sHasher>, _>>::groth_params(
+                &compound_public_params.vanilla_params,
+                &JJ_PARAMS,
+            )?;
 
         let multi_proof = {
             let FuncMeasurement {

filecoin-proofs/Cargo.toml

@@ -39,6 +39,8 @@ os_type = "2.2.0"
 flate2 = { version = "1.0.9", features = ["rust_backend"]}
 tar = "0.4.26"
 rayon = "1.1.0"
+blake2s_simd = "0.5.8"
+hex = "0.4.0"
 
 [dependencies.reqwest]
 version = "0.9"
@@ -49,6 +51,8 @@ features = ["default-tls-vendored"]
 gperftools = "0.2"
 criterion = "0.2"
 rexpect = "0.3.0"
+pretty_env_logger = "0.3.1"
+pretty_assertions = "0.6.1"
 
 [features]
 default = []

filecoin-proofs/benches/preprocessing.rs

@@ -27,9 +27,9 @@ fn preprocessing_benchmark(c: &mut Criterion) {
                 let data = random_data(*size);
 
                 b.iter(|| {
-                    let mut tmpfile: File = tempfile::tempfile().unwrap();
+                    let tmpfile: File = tempfile::tempfile().unwrap();
 
-                    write_padded_bench(&mut tmpfile, data.clone());
+                    write_padded_bench(tmpfile, data.clone());
                 })
             },
             vec![128, 256, 512, 256_000, 512_000, 1024_000, 2048_000],
@@ -38,9 +38,9 @@ fn preprocessing_benchmark(c: &mut Criterion) {
             let data = random_data(*size);
 
             b.iter(|| {
-                let mut tmpfile: File = tempfile::tempfile().unwrap();
+                let tmpfile: File = tempfile::tempfile().unwrap();
 
-                write_padded_unpadded_bench(&mut tmpfile, data.clone());
+                write_padded_unpadded_bench(tmpfile, data.clone());
             })
         })
         .sample_size(2)
@@ -49,15 +49,15 @@ fn preprocessing_benchmark(c: &mut Criterion) {
     );
 }
 
-fn write_padded_bench(file: &mut File, data: Vec<u8>) {
-    let _ = write_padded(&mut &data[..], file).unwrap();
+fn write_padded_bench(mut file: File, data: Vec<u8>) {
+    let _ = write_padded(&mut &data[..], &mut file).unwrap();
     let padded_written = file.seek(SeekFrom::End(0)).unwrap() as usize;
 
     assert!(padded_written > data.len());
 }
 
-fn write_padded_unpadded_bench(file: &mut File, data: Vec<u8>) {
-    write_padded(&mut &data[..], file).unwrap();
+fn write_padded_unpadded_bench(mut file: File, data: Vec<u8>) {
+    write_padded(&mut &data[..], &mut file).unwrap();
 
     let padded_written = file.seek(SeekFrom::End(0)).unwrap() as usize;
 

filecoin-proofs/examples/stacked.rs

@@ -125,7 +125,7 @@ pub fn file_backed_mmap_from(data: &[u8]) -> MmapMut {
     unsafe { MmapOptions::new().map_mut(&tmpfile).unwrap() }
 }
 
-fn dump_proof_bytes<H: Hasher>(all_partition_proofs: &[Vec<stacked::Proof<H>>]) {
+fn dump_proof_bytes<H: Hasher, G: Hasher>(all_partition_proofs: &[Vec<stacked::Proof<H, G>>]) {
     let file = OpenOptions::new()
         .write(true)
         .create(true)
@@ -180,7 +180,7 @@ fn do_the_work<H: 'static>(
 
     info!("running setup");
     start_profile("setup");
-    let pp = StackedDrg::<H>::setup(&sp).unwrap();
+    let pp = StackedDrg::<H, Blake2sHasher>::setup(&sp).unwrap();
     info!("setup complete");
     stop_profile();
 
@@ -196,16 +196,17 @@ fn do_the_work<H: 'static>(
         let mut replication_duration = Duration::new(0, 0);
 
         info!("running replicate");
+        let seed = rng.gen();
 
         start_profile("replicate");
         let (tau, (p_aux, t_aux)) =
-            StackedDrg::<H>::replicate(&pp, &replica_id, &mut data, None).unwrap();
+            StackedDrg::<H, Blake2sHasher>::replicate(&pp, &replica_id, &mut data, None).unwrap();
         stop_profile();
-        let pub_inputs = stacked::PublicInputs::<H::Domain> {
+        let pub_inputs = stacked::PublicInputs::<H::Domain, <Blake2sHasher as Hasher>::Domain> {
             replica_id,
             tau: Some(tau.clone()),
             k: Some(0),
-            seed: None,
+            seed,
         };
 
         let priv_inputs = stacked::PrivateInputs { p_aux, t_aux };
@@ -233,9 +234,13 @@ fn do_the_work<H: 'static>(
 
         let start = Instant::now();
         start_profile("prove");
-        let all_partition_proofs =
-            StackedDrg::<H>::prove_all_partitions(&pp, &pub_inputs, &priv_inputs, partitions)
-                .expect("failed to prove");
+        let all_partition_proofs = StackedDrg::<H, Blake2sHasher>::prove_all_partitions(
+            &pp,
+            &pub_inputs,
+            &priv_inputs,
+            partitions,
+        )
+        .expect("failed to prove");
         stop_profile();
         let vanilla_proving = start.elapsed();
         total_proving += vanilla_proving;
@@ -251,9 +256,12 @@ fn do_the_work<H: 'static>(
         start_profile("verify");
         for _ in 0..samples {
             let start = Instant::now();
-            let verified =
-                StackedDrg::<H>::verify_all_partitions(&pp, &pub_inputs, &all_partition_proofs)
-                    .expect("failed during verification");
+            let verified = StackedDrg::<H, Blake2sHasher>::verify_all_partitions(
+                &pp,
+                &pub_inputs,
+                &all_partition_proofs,
+            )
+            .expect("failed during verification");
             if !verified {
                 info!("Verification failed.");
             };
@@ -283,9 +291,11 @@ fn do_the_work<H: 'static>(
             info!("Performing circuit bench.");
             let mut cs = MetricCS::<Bls12>::new();
 
-            StackedCompound::blank_circuit(&pp, &JJ_PARAMS)
-                .synthesize(&mut cs)
-                .expect("failed to synthesize circuit");
+            <StackedCompound as CompoundProof<_, StackedDrg<H, Blake2sHasher>, _>>::blank_circuit(
+                &pp, &JJ_PARAMS,
+            )
+            .synthesize(&mut cs)
+            .expect("failed to synthesize circuit");
 
             info!("circuit_num_inputs: {}", cs.num_inputs());
             info!("circuit_num_constraints: {}", cs.num_constraints());
@@ -307,7 +317,7 @@ fn do_the_work<H: 'static>(
             // and skip replication/vanilla-proving entirely.
             info!("Performing circuit groth.");
             let gparams =
-                StackedCompound::groth_params(&compound_public_params.vanilla_params, &JJ_PARAMS)
+                <StackedCompound as CompoundProof<_, StackedDrg<H, Blake2sHasher>, _>>::groth_params(&compound_public_params.vanilla_params, &JJ_PARAMS)
                     .unwrap();
 
             let multi_proof = {
@@ -365,7 +375,8 @@ fn do_the_work<H: 'static>(
             let start = Instant::now();
             info!("Extracting.");
             start_profile("extract");
-            let decoded_data = StackedDrg::<H>::extract_all(&pp, &replica_id, &data).unwrap();
+            let decoded_data =
+                StackedDrg::<H, Blake2sHasher>::extract_all(&pp, &replica_id, &data).unwrap();
             stop_profile();
             let extracting = start.elapsed();
             info!("extracting_time: {:?}", extracting);