Decode snappy-compressed files

Author: Mark-Simulacrum

Cargo.lock

@@ -29,6 +29,7 @@ jemallocator = "0.3"
 jemalloc-ctl = "0.3"
 rust_team_data = { git = "https://github.com/rust-lang/team" }
 parking_lot = "0.9"
+snap = "0.2.5"
 
 [dependencies.collector]
 path = "../collector"

site/Cargo.toml

@@ -193,7 +193,7 @@ impl InputData {
         let repo_loc = PathBuf::from(repo_loc);
         let mut skipped = 0;
         let mut artifact_data = BTreeMap::new();
-        let mut data = BTreeMap::new();
+        let mut data = HashMap::new();
 
         if !repo_loc.exists() {
             // If the repository doesn't yet exist, simplify clone it to the given location.
@@ -214,19 +214,36 @@ impl InputData {
         trace!("loading files from directory");
 
         // Read all files from repo_loc/processed
-        let mut file_count = 0;
+        let mut files = Vec::new();
         for entry in fs::read_dir(repo_loc.join("times"))? {
             let entry = entry?;
             if entry.file_type()?.is_dir() {
                 continue;
             }
-            file_count += 1;
-
             let filename = entry.file_name();
             let filename = filename.to_str().unwrap();
             let file_contents =
                 fs::read(entry.path()).with_context(|| format!("Failed to read {}", filename))?;
 
+            files.push((filename.to_owned(), file_contents));
+        }
+
+        trace!("read directory");
+
+        data.reserve(files.len());
+        for (filename, file_contents) in &files {
+            let c;
+            let file_contents = if filename.ends_with(".sz") {
+                use std::io::Read;
+                let mut out = Vec::with_capacity(snap::decompress_len(&file_contents).unwrap_or(0));
+                let mut szip_reader = snap::Reader::new(&file_contents[..]);
+                szip_reader.read_to_end(&mut out).unwrap();
+                c = out;
+                &c
+            } else {
+                &file_contents
+            };
+
             if filename.starts_with("artifact-") {
                 let contents: ArtifactData = match serde_json::from_slice(&file_contents) {
                     Ok(j) => j,
@@ -262,7 +279,7 @@ impl InputData {
             }
         }
 
-        info!("{} total files", file_count);
+        info!("{} total files", files.len());
         info!("{} skipped files", skipped);
         info!("{} measured", data.len());
 
@@ -275,7 +292,7 @@ impl InputData {
             }
         };
 
-        InputData::new(data, artifact_data, config)
+        InputData::new(data.into_iter().collect(), artifact_data, config)
     }
 
     pub fn new(