Auto merge of #376 - pietroalbini:crates-build-env, r=pietroalbini

Switch build environment to rustops/crates-build-env

This PR switches the build environment used by Crater to the Docker image built in the [rust-ops/crates-build-env](https://github.com/rust-ops/crates-build-env) repository. Instead of installing GTK and all the suggested stuff and hoping for the best, the new image installs a curated set of packages built over time by the docs.rs developers (thanks @onur!) which weights about 3GB (1GB compressed download).

The image is also downloaded from Docker Hub instead of being built locally, and it's automatically updated every time a run is started. This also means `prepare-local` is no longer needed if you only want to execute a run (creating new ones still requires the command).

This PR also removes the distinction between the mini and the full images, since the new image has a reasonable size for local development and it's prebuilt.

Author: bors

ci/run/minicrater-linux.sh

@@ -2,5 +2,5 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-cargo run -- prepare-local --docker-env=mini
+cargo run -- create-lists
 MINICRATER_SHOW_OUTPUT=1 cargo test -- --ignored --nocapture --test-threads 1

docker/Dockerfile.full

@@ -10,7 +10,7 @@ of its output is into the `./work` directory, where it maintains its
 own rustup installation, crate mirrors, etc.
 
 ```
-cargo run -- prepare-local --docker-env mini
+cargo run -- prepare-local
 cargo run -- define-ex --crate-select=demo --cap-lints=forbid stable beta
 cargo run -- run-graph --threads NUM_CPUS
 cargo run -- gen-report work/ex/default/

docker/Dockerfile.mini

@@ -47,15 +47,15 @@ fn run_heartbeat(url: &str, token: &str) {
     });
 }
 
-pub fn run(url: &str, token: &str, threads_count: usize) -> Fallible<()> {
+pub fn run(url: &str, token: &str, threads_count: usize, docker_env: &str) -> Fallible<()> {
     let agent = Agent::new(url, token)?;
     let db = results::ResultsUploader::new(&agent.api);
 
     run_heartbeat(url, token);
 
     loop {
         let ex = agent.experiment()?;
-        ::runner::run_ex(&ex, &db, threads_count, &agent.config)?;
+        ::runner::run_ex(&ex, &db, threads_count, &agent.config, docker_env)?;
         agent.api.complete_experiment()?;
     }
 }

docker/run2.sh

@@ -14,7 +14,6 @@ use crater::agent;
 use crater::config::Config;
 use crater::crates::Crate;
 use crater::db::Database;
-use crater::docker;
 use crater::experiments::{Assignee, CapLints, CrateSelect, Experiment, Mode, Status};
 use crater::report;
 use crater::results::{DatabaseDB, DeleteResults};
@@ -27,6 +26,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use structopt::clap::AppSettings;
 
+static DEFAULT_DOCKER_ENV: &str = "rustops/crates-build-env";
+
 // An experiment name
 #[derive(Debug, Clone)]
 pub struct Ex(String);
@@ -75,10 +76,7 @@ pub enum Crater {
         name = "prepare-local",
         about = "acquire toolchains, build containers, build crate lists"
     )]
-    PrepareLocal {
-        #[structopt(name = "docker env", long = "docker-env", default_value = "full")]
-        env: DockerEnv,
-    },
+    PrepareLocal,
 
     #[structopt(name = "create-lists", about = "create all the lists of crates")]
     CreateLists {
@@ -189,6 +187,8 @@ pub enum Crater {
         ex: Ex,
         #[structopt(name = "threads", short = "t", long = "threads", default_value = "1")]
         threads: usize,
+        #[structopt(name = "docker-env", long = "docker-env")]
+        docker_env: Option<String>,
     },
 
     #[structopt(name = "gen-report", about = "generate the experiment report")]
@@ -227,6 +227,8 @@ pub enum Crater {
         token: String,
         #[structopt(name = "threads", short = "t", long = "threads", default_value = "1")]
         threads: usize,
+        #[structopt(name = "docker-env", long = "docker-env")]
+        docker_env: Option<String>,
     },
 
     #[structopt(
@@ -275,12 +277,9 @@ impl Crater {
                     action.apply(&db, &config)?;
                 }
             }
-            Crater::PrepareLocal { ref env } => {
+            Crater::PrepareLocal => {
                 let config = Config::load()?;
                 let db = Database::open()?;
-
-                let docker_env = &env.0;
-                docker::build_container(docker_env)?;
                 actions::UpdateLists::default().apply(&db, &config)?;
             }
             Crater::DefineEx {
@@ -367,7 +366,15 @@ impl Crater {
                     bail!("missing experiment {}", ex.0);
                 }
             }
-            Crater::RunGraph { ref ex, threads } => {
+            Crater::RunGraph {
+                ref ex,
+                threads,
+                ref docker_env,
+            } => {
+                let docker_env = docker_env
+                    .as_ref()
+                    .map(|e| e.as_str())
+                    .unwrap_or(DEFAULT_DOCKER_ENV);
                 let config = Config::load()?;
                 let db = Database::open()?;
 
@@ -387,7 +394,7 @@ impl Crater {
                     }
 
                     let result_db = DatabaseDB::new(&db);
-                    runner::run_ex(&experiment, &result_db, threads, &config)?;
+                    runner::run_ex(&experiment, &result_db, threads, &config, docker_env)?;
                     experiment.set_status(&db, Status::NeedsReport)?;
                 } else {
                     bail!("missing experiment {}", ex.0);
@@ -481,8 +488,13 @@ impl Crater {
                 ref url,
                 ref token,
                 threads,
+                ref docker_env,
             } => {
-                agent::run(url, token, threads)?;
+                let docker_env = docker_env
+                    .as_ref()
+                    .map(|e| e.as_str())
+                    .unwrap_or(DEFAULT_DOCKER_ENV);
+                agent::run(url, token, threads, docker_env)?;
             }
             Crater::DumpTasksGraph { ref dest, ref ex } => {
                 let config = Config::load()?;

docs/cli-usage.md

@@ -6,21 +6,51 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use utils::size::Size;
 
-pub(crate) static IMAGE_NAME: &'static str = "crater";
-
-/// Builds the docker container image, 'crater', what will be used
-/// to isolate builds from each other. This expects the Dockerfile
-/// to exist in the `docker` directory, at runtime.
-pub fn build_container(docker_env: &str) -> Fallible<()> {
-    let dockerfile = format!("docker/Dockerfile.{}", docker_env);
+pub(crate) fn is_running() -> bool {
+    info!("checking if the docker daemon is running");
     RunCommand::new("docker")
-        .args(&["build", "-f", &dockerfile, "-t", IMAGE_NAME, "docker"])
-        .enable_timeout(false)
+        .args(&["info"])
+        .hide_output(true)
         .run()
+        .is_ok()
 }
 
-pub(crate) fn is_running() -> bool {
-    RunCommand::new("docker").args(&["info"]).run().is_ok()
+pub(crate) struct DockerEnv {
+    image: String,
+    local: bool,
+}
+
+impl DockerEnv {
+    pub(crate) fn new(image: &str) -> Self {
+        DockerEnv {
+            image: image.to_string(),
+            local: !image.contains('/'),
+        }
+    }
+
+    pub(crate) fn ensure_exists_locally(&self) -> Fallible<()> {
+        if !self.local {
+            self.pull()?;
+        } else {
+            info!("docker environment is local, skipping pull");
+        }
+
+        info!("checking the image {} is available locally", self.image);
+        RunCommand::new("docker")
+            .args(&["image", "inspect", &self.image])
+            .hide_output(true)
+            .run()?;
+
+        Ok(())
+    }
+
+    fn pull(&self) -> Fallible<()> {
+        info!("pulling image {} from Docker Hub", self.image);
+        RunCommand::new("docker")
+            .args(&["pull", &self.image])
+            .enable_timeout(false)
+            .run()
+    }
 }
 
 #[derive(Copy, Clone)]
@@ -50,21 +80,25 @@ impl MountConfig {
     }
 }
 
-pub(crate) struct ContainerBuilder {
-    image: String,
+pub(crate) struct ContainerBuilder<'a> {
+    image: &'a DockerEnv,
     mounts: Vec<MountConfig>,
     env: Vec<(String, String)>,
     memory_limit: Option<Size>,
+    workdir: Option<String>,
+    cmd: Vec<String>,
     enable_networking: bool,
 }
 
-impl ContainerBuilder {
-    pub(crate) fn new<S: Into<String>>(image: S) -> Self {
+impl<'a> ContainerBuilder<'a> {
+    pub(crate) fn new(image: &'a DockerEnv) -> Self {
         ContainerBuilder {
-            image: image.into(),
+            image,
             mounts: Vec::new(),
             env: Vec::new(),
+            workdir: None,
             memory_limit: None,
+            cmd: Vec::new(),
             enable_networking: true,
         }
     }
@@ -88,11 +122,21 @@ impl ContainerBuilder {
         self
     }
 
+    pub(crate) fn workdir<S: Into<String>>(mut self, workdir: S) -> Self {
+        self.workdir = Some(workdir.into());
+        self
+    }
+
     pub(crate) fn memory_limit(mut self, limit: Option<Size>) -> Self {
         self.memory_limit = limit;
         self
     }
 
+    pub(crate) fn cmd(mut self, cmd: Vec<String>) -> Self {
+        self.cmd = cmd;
+        self
+    }
+
     pub(crate) fn enable_networking(mut self, enable: bool) -> Self {
         self.enable_networking = enable;
         self
@@ -112,6 +156,11 @@ impl ContainerBuilder {
             args.push(format! {"{}={}", var, value})
         }
 
+        if let Some(workdir) = self.workdir {
+            args.push("-w".into());
+            args.push(workdir);
+        }
+
         if let Some(limit) = self.memory_limit {
             args.push("-m".into());
             args.push(limit.to_string());
@@ -122,7 +171,11 @@ impl ContainerBuilder {
             args.push("none".into());
         }
 
-        args.push(self.image);
+        args.push(self.image.image.clone());
+
+        for arg in self.cmd {
+            args.push(arg);
+        }
 
         let (out, _) = RunCommand::new("docker").args(&*args).run_capture()?;
         Ok(Container { id: out[0].clone() })

src/agent/mod.rs

@@ -72,7 +72,7 @@ pub mod config;
 pub mod crates;
 pub mod db;
 pub mod dirs;
-pub mod docker;
+mod docker;
 pub mod experiments;
 mod native;
 mod prelude;