diff --git a/Cargo.lock b/Cargo.lock index 9749049d1..1da72242a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,6 +428,7 @@ dependencies = [ "env_logger", "failure", "flate2", + "git2", "hmac 0.7.1", "http 0.1.21", "hyper 0.12.35", @@ -1028,9 +1029,9 @@ checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" [[package]] name = "git2" -version = "0.13.12" +version = "0.13.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6f1a0238d7f8f8fd5ee642f4ebac4dbc03e03d1f78fbe7a3ede35dcf7e2224" +checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" dependencies = [ "bitflags", "libc", @@ -1481,9 +1482,9 @@ checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "libgit2-sys" -version = "0.12.14+1.1.0" +version = "0.12.26+1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f25af58e6495f7caf2919d08f212de550cfa3ed2f5e744988938ea292b9f549" +checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 822657cc5..1990754b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ ctrlc = "3.1.3" prometheus = "0.7.0" cargo_metadata = "0.12.1" indexmap = "1.4.0" +git2 = { version = "0.13", features = ["vendored-libgit2"] } [dev-dependencies] assert_cmd = "1.0.2" diff --git a/config.toml b/config.toml index df959f59d..06475de29 100644 --- a/config.toml +++ b/config.toml @@ -27,7 +27,8 @@ local-crates = [] [sandbox] # Maximum amount of RAM allowed during builds -memory-limit = "1536M" # 1.5G +memory-limit = "4096M" # 4G, 1G/cpu +cpu-limit = 4 # Restrictions on the amount of information stored in build logs build-log-max-size = "5M" build-log-max-lines = 10000 diff --git a/src/config.rs b/src/config.rs index ae637837b..d36d085b2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -74,6 +74,7 @@ pub struct DemoCrates { #[serde(rename_all = "kebab-case")] pub struct SandboxConfig { pub memory_limit: Size, + pub cpu_limit: u16, pub build_log_max_size: Size, pub build_log_max_lines: usize, } @@ -256,6 +257,7 @@ impl Default for Config { local_crates: HashMap::new(), sandbox: SandboxConfig { memory_limit: Size::Gigabytes(2), + cpu_limit: 1, build_log_max_size: Size::Megabytes(1), build_log_max_lines: 1000, }, @@ -308,6 +310,7 @@ mod tests { "local-crates = []\n", "[sandbox]\n", "memory-limit = \"2G\"\n", + "cpu-limit = 1\n", "build-log-max-size = \"2M\"\n", "build-log-max-lines = 1000\n", "[crates]\n", diff --git a/src/runner/test.rs b/src/runner/test.rs index f1473095d..de31e914e 100644 --- a/src/runner/test.rs +++ b/src/runner/test.rs @@ -89,6 +89,9 @@ fn run_cargo( rustflags.push(' '); rustflags.push_str(tc_rustflags); } + // Limit debuginfo in an effort to decrease linker peak memory usage, while retaining enough + // information to get useful backtraces. + rustflags.push_str(" -Cdebuginfo=1"); let rustflags_env = if let Some(&"doc") = args.get(0) { "RUSTDOCFLAGS" @@ -210,6 +213,7 @@ pub(super) fn run_test( ); let sandbox = SandboxBuilder::new() .memory_limit(Some(ctx.config.sandbox.memory_limit.to_bytes())) + .cpu_limit(Some(f32::from(ctx.config.sandbox.cpu_limit))) .enable_networking(false); let krate = &ctx.krate.to_rustwide(); @@ -235,17 +239,31 @@ fn build( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible<()> { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); run_cargo( ctx, build_env, - &["build", "--frozen", "--message-format=json"], + &[ + "build", + "--frozen", + "--message-format=json", + "--jobs", + &cpus, + ], true, local_packages_id, )?; run_cargo( ctx, build_env, - &["test", "--frozen", "--no-run", "--message-format=json"], + &[ + "test", + "--frozen", + "--no-run", + "--message-format=json", + "--jobs", + &cpus, + ], true, local_packages_id, )?; @@ -253,10 +271,19 @@ fn build( } fn test(ctx: &TaskCtx, build_env: &Build) -> Fallible<()> { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); run_cargo( ctx, build_env, - &["test", "--frozen"], + &[ + "test", + "--frozen", + "--jobs", + &cpus, + "--", + "--test-threads", + &cpus, + ], false, &HashSet::new(), ) @@ -299,6 +326,7 @@ pub(super) fn test_check_only( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); if let Err(err) = run_cargo( ctx, build_env, @@ -308,6 +336,8 @@ pub(super) fn test_check_only( "--all", "--all-targets", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, @@ -323,6 +353,7 @@ pub(super) fn test_clippy_only( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); if let Err(err) = run_cargo( ctx, build_env, @@ -332,6 +363,8 @@ pub(super) fn test_clippy_only( "--all", "--all-targets", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, @@ -347,6 +380,7 @@ pub(super) fn test_rustdoc( build_env: &Build, local_packages_id: &HashSet, ) -> Fallible { + let cpus = ctx.config.sandbox.cpu_limit.to_string(); let res = run_cargo( ctx, build_env, @@ -356,6 +390,8 @@ pub(super) fn test_rustdoc( "--no-deps", "--document-private-items", "--message-format=json", + "--jobs", + &cpus, ], true, local_packages_id, diff --git a/tests/check_config/bad-duplicate-crate.toml b/tests/check_config/bad-duplicate-crate.toml index f4e438e0d..b2e553750 100644 --- a/tests/check_config/bad-duplicate-crate.toml +++ b/tests/check_config/bad-duplicate-crate.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/bad-missing-crate.toml b/tests/check_config/bad-missing-crate.toml index 3fff66502..d5d2f6d36 100644 --- a/tests/check_config/bad-missing-crate.toml +++ b/tests/check_config/bad-missing-crate.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/bad-missing-repo.toml b/tests/check_config/bad-missing-repo.toml index 6b077bc3b..7477fee72 100644 --- a/tests/check_config/bad-missing-repo.toml +++ b/tests/check_config/bad-missing-repo.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit= 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/check_config/good.toml b/tests/check_config/good.toml index 4f9903b27..1de068867 100644 --- a/tests/check_config/good.toml +++ b/tests/check_config/good.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "1536M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/blacklist/config.toml b/tests/minicrater/blacklist/config.toml index 7a04174b8..a349ade06 100644 --- a/tests/minicrater/blacklist/config.toml +++ b/tests/minicrater/blacklist/config.toml @@ -18,6 +18,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/clippy/config.toml b/tests/minicrater/clippy/config.toml index c64180af7..6ec5404fd 100644 --- a/tests/minicrater/clippy/config.toml +++ b/tests/minicrater/clippy/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "clippy-warn"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/full/config.toml b/tests/minicrater/full/config.toml index b138311b9..f23f1763b 100644 --- a/tests/minicrater/full/config.toml +++ b/tests/minicrater/full/config.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/ignore-blacklist/config.toml b/tests/minicrater/ignore-blacklist/config.toml index 973f0be1b..2d96b5b90 100644 --- a/tests/minicrater/ignore-blacklist/config.toml +++ b/tests/minicrater/ignore-blacklist/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "build-fail", "test-fail"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/missing-repo/config.toml b/tests/minicrater/missing-repo/config.toml index 7f509facc..407f36266 100644 --- a/tests/minicrater/missing-repo/config.toml +++ b/tests/minicrater/missing-repo/config.toml @@ -17,6 +17,7 @@ local-crates = [] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/resource-exhaustion/config.toml b/tests/minicrater/resource-exhaustion/config.toml index e74994989..799c127e7 100644 --- a/tests/minicrater/resource-exhaustion/config.toml +++ b/tests/minicrater/resource-exhaustion/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "memory-hungry"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000 diff --git a/tests/minicrater/small/config.toml b/tests/minicrater/small/config.toml index fcedc7cfb..98f5160b4 100644 --- a/tests/minicrater/small/config.toml +++ b/tests/minicrater/small/config.toml @@ -17,6 +17,7 @@ local-crates = ["build-pass", "beta-regression"] [sandbox] memory-limit = "512M" +cpu-limit = 1 build-log-max-size = "2M" build-log-max-lines = 1000