Update decoding to stream into a single pre-allocated buffer

Author: Mark-Simulacrum

Cargo.lock

@@ -162,10 +162,11 @@ impl Ord for Commit {
 pub struct Patch {
     index: usize,
     pub name: PatchName,
-    path: PathBuf,
+    path: PatchPath,
 }
 
 crate::intern!(pub struct PatchName);
+crate::intern!(pub struct PatchPath);
 
 impl PartialEq for Patch {
     fn eq(&self, other: &Self) -> bool {
@@ -207,7 +208,7 @@ impl Patch {
         };
 
         Patch {
-            path: PathBuf::from(path.file_name().unwrap()),
+            path: PatchPath::from(path.file_name().unwrap().to_str().unwrap()),
             index,
             name: name.as_str().into(),
         }
@@ -216,7 +217,7 @@ impl Patch {
     pub fn apply(&self, dir: &Path) -> Result<(), String> {
         log::debug!("applying {} to {:?}", self.name, dir);
         let mut cmd = process::Command::new("patch");
-        cmd.current_dir(dir).args(&["-Np1", "-i"]).arg(&self.path);
+        cmd.current_dir(dir).args(&["-Np1", "-i"]).arg(&*self.path);
         cmd.stdout(Stdio::null());
         if cmd.status().map(|s| !s.success()).unwrap_or(false) {
             return Err(format!("could not execute {:?}.", cmd));
@@ -269,7 +270,7 @@ impl BenchmarkState {
         match &mut self {
             BenchmarkState::IncrementalPatched(patch) => {
                 patch.index = 0;
-                patch.path = PathBuf::new();
+                patch.path = PatchPath::from("");
             }
             _ => {}
         }

collector/src/lib.rs

@@ -30,7 +30,7 @@ jemallocator = "0.3"
 jemalloc-ctl = "0.3"
 rust_team_data = { git = "https://github.com/rust-lang/team" }
 parking_lot = "0.10"
-snap = "0.2.5"
+snap = "1"
 rustc-artifacts = "0.1.1"
 
 [dependencies.collector]

site/Cargo.toml

@@ -10,6 +10,7 @@
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::env;
 use std::fs;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -216,27 +217,27 @@ impl InputData {
 
         // Read all files from repo_loc/processed
         let latest_section_start = ::std::time::Instant::now();
+        let mut file_contents = Vec::new();
         for entry in fs::read_dir(repo_loc.join("times"))? {
             let entry = entry?;
             if entry.file_type()?.is_dir() {
                 continue;
             }
             let filename = entry.file_name();
             let filename = filename.to_str().unwrap();
-            let file_contents =
-                fs::read(entry.path()).with_context(|| format!("Failed to read {}", filename))?;
-            let c;
-            let file_contents = if filename.ends_with(".sz") {
-                use std::io::Read;
-                let mut out =
-                    String::with_capacity(snap::decompress_len(&file_contents).unwrap_or(0));
-                let mut szip_reader = snap::Reader::new(&file_contents[..]);
-                szip_reader.read_to_string(&mut out).unwrap();
-                c = out;
-                c.as_str()
+            let mut file = fs::File::open(entry.path())
+                .with_context(|| format!("Failed to open {}", entry.path().display()))?;
+            file_contents.truncate(0);
+            if filename.ends_with(".sz") {
+                let mut szip_reader = snap::read::FrameDecoder::new(std::io::BufReader::new(file));
+                szip_reader
+                    .read_to_end(&mut file_contents)
+                    .with_context(|| format!("Failed to read {}", entry.path().display()))?;
             } else {
-                std::str::from_utf8(&file_contents).unwrap()
+                file.read_to_end(&mut file_contents)
+                    .with_context(|| format!("Failed to read {}", entry.path().display()))?;
             };
+            let file_contents = std::str::from_utf8(&file_contents).unwrap();
 
             if filename.starts_with("artifact-") {
                 let contents: ArtifactData = match serde_json::from_str(&file_contents) {
@@ -270,6 +271,7 @@ impl InputData {
                 }
             }
         }
+        std::mem::drop(file_contents);
 
         info!(
             "{} commits/artifacts loaded in {:?}",