Merge pull request #26 from rust-lang/purge-caches

Add the Workspace::purge_all_caches method

Author: pietroalbini

CHANGELOG.md

@@ -5,6 +5,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- New method `Workspace::purge_all_caches`
+
 ### Changed
 
 - Subcommands executed in sandbox respect configs of parent command

src/workspace.rs

@@ -220,6 +220,31 @@ impl Workspace {
         Ok(())
     }
 
+    /// Remove all the contents of the caches in the workspace, freeing disk space.
+    pub fn purge_all_caches(&self) -> Result<(), Error> {
+        let mut paths = vec![
+            self.cache_dir(),
+            self.cargo_home().join("git"),
+            self.cargo_home().join("registry").join("src"),
+            self.cargo_home().join("registry").join("cache"),
+        ];
+
+        for index in std::fs::read_dir(self.cargo_home().join("registry").join("index"))? {
+            let index = index?;
+            if index.file_type()?.is_dir() {
+                paths.push(index.path().join(".cache"));
+            }
+        }
+
+        for path in &paths {
+            if path.exists() {
+                remove_dir_all(&path)?;
+            }
+        }
+
+        Ok(())
+    }
+
     /// Return a list of all the toolchains present in the workspace.
     ///
     /// # Example

tests/buildtest/crates/process-lines/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "process-lines"
+version = "0.1.0"
+authors = ["Pietro Albini <pietro@pietroalbini.org>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

tests/buildtest/crates/process-lines/src/main.rs

@@ -0,0 +1,4 @@
+fn main() {
+    println!("Hello, world!");
+    println!("Hello, world again!");
+}

tests/buildtest/mod.rs

@@ -28,7 +28,7 @@ fn test_hello_world() {
 
 #[test]
 fn test_process_lines() {
-    runner::run("hello-world", |run| {
+    runner::run("process-lines", |run| {
         run.build(SandboxBuilder::new().enable_networking(false), |build| {
             let storage = rustwide::logging::LogStorage::new(LevelFilter::Info);
             let mut ex = false;

tests/integration/mod.rs

@@ -1 +1,2 @@
 mod crates_git;
+mod purge_caches;

tests/integration/purge_caches.rs

@@ -0,0 +1,131 @@
+use failure::Error;
+use rustwide::cmd::SandboxBuilder;
+use rustwide::{Crate, Toolchain};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+const WORKSPACE_NAME: &str = "purge-caches";
+
+#[test]
+fn test_purge_caches() -> Result<(), Error> {
+    let workspace_path = crate::utils::workspace_path(WORKSPACE_NAME);
+    let workspace = crate::utils::init_named_workspace(WORKSPACE_NAME)?;
+
+    // Do an initial purge to prevent stale files from being present.
+    workspace.purge_all_build_dirs()?;
+    workspace.purge_all_caches()?;
+
+    let toolchain = Toolchain::dist("stable");
+    toolchain.install(&workspace)?;
+
+    let start_contents = WorkspaceContents::collect(&workspace_path)?;
+
+    let crates = vec![
+        Crate::crates_io("lazy_static", "1.0.0"),
+        Crate::git("https://github.com/pietroalbini/git-credential-null"),
+    ];
+
+    // Simulate a build, which is going to fill up the caches.
+    for krate in &crates {
+        krate.fetch(&workspace)?;
+
+        let sandbox = SandboxBuilder::new().enable_networking(false);
+        let mut build_dir = workspace.build_dir("shared");
+        build_dir.build(&toolchain, krate, sandbox).run(|build| {
+            build.cargo().args(&["check"]).run()?;
+            Ok(())
+        })?;
+    }
+
+    // After all the builds are done purge everything again, and ensure the contents are the same
+    // as when we started.
+    workspace.purge_all_build_dirs()?;
+    workspace.purge_all_caches()?;
+    let end_contents = WorkspaceContents::collect(&workspace_path)?;
+    start_contents.assert_same(end_contents);
+
+    Ok(())
+}
+
+/// Define which files should be ignored when comparing the two workspaces. If there are expected
+/// changes, update the function to match them.
+fn should_compare(base: &Path, path: &Path) -> bool {
+    let components = match path.strip_prefix(base) {
+        Ok(stripped) => stripped
+            .components()
+            .map(|component| component.as_os_str().to_string_lossy().to_string())
+            .collect::<Vec<_>>(),
+        Err(_) => return true,
+    };
+
+    let components = components.iter().map(|c| c.as_str()).collect::<Vec<_>>();
+    match components.as_slice() {
+        // The indexes could be updated during the build. The index is not considered a cache
+        // though, so it's fine to ignore it during the comparison.
+        ["cargo-home", "registry", "index", _, ".git", ..] => false,
+        ["cargo-home", "registry", "index", _, ".cargo-index-lock"] => false,
+        ["cargo-home", "registry", "index", _, ".last-updated"] => false,
+
+        _ => true,
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct WorkspaceContents {
+    base: PathBuf,
+    files: HashMap<PathBuf, u64>,
+}
+
+impl WorkspaceContents {
+    fn collect(path: &Path) -> Result<Self, Error> {
+        let mut files = HashMap::new();
+
+        for entry in walkdir::WalkDir::new(path)
+            .into_iter()
+            .filter_entry(|entry| should_compare(path, entry.path()))
+        {
+            let entry = entry?;
+            let metadata = entry.metadata()?;
+
+            if !metadata.is_file() {
+                continue;
+            }
+
+            files.insert(entry.path().into(), metadata.len());
+        }
+
+        Ok(Self {
+            base: path.into(),
+            files,
+        })
+    }
+
+    fn assert_same(self, mut other: Self) {
+        let mut same = true;
+
+        println!("=== start directory differences ===");
+
+        for (path, start_digest) in self.files.into_iter() {
+            if let Some(end_digest) = other.files.remove(&path) {
+                if start_digest != end_digest {
+                    println!("file {} changed its size", path.display());
+                    same = false;
+                }
+            } else {
+                println!("file {} was removed", path.display());
+                same = false;
+            }
+        }
+
+        for (path, _) in other.files.into_iter() {
+            println!("file {} was added", path.display());
+            same = false;
+        }
+
+        println!("=== end directory differences ===");
+
+        if !same {
+            panic!("the contents of the directory changed");
+        }
+    }
+}

tests/utils/mod.rs

@@ -1,13 +1,21 @@
 use failure::Error;
 use log::LevelFilter;
 use rustwide::{Workspace, WorkspaceBuilder};
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 static USER_AGENT: &str = "rustwide-tests (https://github.com/rust-lang/rustwide)";
 
+pub(crate) fn workspace_path(name: &str) -> PathBuf {
+    Path::new(".workspaces").join(name)
+}
+
 pub(crate) fn init_workspace() -> Result<Workspace, Error> {
+    init_named_workspace("integration")
+}
+
+pub(crate) fn init_named_workspace(name: &str) -> Result<Workspace, Error> {
     init_logs();
-    let workspace_path = Path::new(".workspaces").join("integration");
+    let workspace_path = workspace_path(name);
     let mut builder = WorkspaceBuilder::new(&workspace_path, USER_AGENT).fast_init(true);
 
     if std::env::var("RUSTWIDE_TEST_INSIDE_DOCKER").is_ok() {