cargo dev crater: work on downloading and extracting crate sources

Author: matthiaskrgr

clippy_dev/Cargo.toml

@@ -1,16 +1,19 @@
 [package]
-name = "clippy_dev"
-version = "0.0.1"
 authors = ["Philipp Hansch <dev@phansch.net>"]
 edition = "2018"
+name = "clippy_dev"
+version = "0.0.1"
 
 [dependencies]
 bytecount = "0.6"
 clap = "2.33"
+flate2 = "1.0.19"
 itertools = "0.9"
 opener = "0.4"
 regex = "1"
 shell-escape = "0.1"
+tar = "0.4.30"
+ureq = "2.0.0-rc3"
 walkdir = "2"
 
 [features]

clippy_dev/src/crater.rs

@@ -2,15 +2,18 @@ use std::path::PathBuf;
 use std::process::Command;
 
 // represents an archive we download from crates.io
+#[derive(Debug)]
 struct KrateSource {
     version: String,
     name: String,
 }
 
 // represents the extracted sourcecode of a crate
+#[derive(Debug)]
 struct Krate {
     version: String,
     name: String,
+    path: PathBuf,
 }
 
 impl KrateSource {
@@ -20,13 +23,34 @@ impl KrateSource {
             name: name.into(),
         }
     }
-    fn download_and_extract(self) -> Krate {
+    fn download_and_extract(&self) -> Krate {
+        let extract_dir = PathBuf::from("target/crater/crates");
+
         // download
+        let krate_download_dir = PathBuf::from("target/crater/downloads");
+
+        let url = format!(
+            "https://crates.io/api/v1/crates/{}/{}/download",
+            self.name, self.version
+        );
+        print!("Downloading {}, {}", self.name, self.version);
+
+        let krate_name = format!("{}-{}.crate", &self.name, &self.version);
+        let mut krate_dest = std::fs::File::create(krate_download_dir.join(krate_name)).unwrap();
+        let mut krate_req = ureq::get(&url).call().unwrap().into_reader();
+        std::io::copy(&mut krate_req, &mut krate_dest).unwrap();
+
         // extract
+        let krate = krate_dest;
+        let tar = flate2::read::GzDecoder::new(krate);
+        let mut archiv = tar::Archive::new(tar);
+        let extracted_path = extract_dir.join(format!("{}-{}/", self.name, self.version));
+        archiv.unpack(&extracted_path).expect("Failed to extract!");
 
         Krate {
-            version: self.version,
-            name: self.name,
+            version: self.version.clone(),
+            name: self.name.clone(),
+            path: extracted_path,
         }
     }
 }
@@ -35,8 +59,6 @@ impl Krate {
     fn run_clippy_lints(&self) -> String {
         todo!();
     }
-
-
 }
 
 fn build_clippy() {
@@ -65,5 +87,10 @@ pub(crate) fn run() {
         "target/debug/clippy-driver binary not found!"
     );
 
-  let clippy_lint_results: Vec<String> = krates.into_iter().map(|krate|  krate.download_and_extract()).map(|krate| krate.run_clippy_lints()).collect::<Vec<String>>();
+    // download and extract the crates, then run clippy on them and collect clippys warnings
+    let clippy_lint_results: Vec<String> = krates
+        .into_iter()
+        .map(|krate| krate.download_and_extract())
+        .map(|krate| krate.run_clippy_lints())
+        .collect::<Vec<String>>();
 }