Add back a full integration test for -Zbuild-std

Only run these tests on one CI builder (not all platforms) though. This
is extremely resource intensive since it rebuilds libstd. Currently this
does not share a build directly like before because the number of tests
are supposed to be small, but if necessary we can add that in later too.

Author: alexcrichton

azure-pipelines.yml

@@ -43,6 +43,7 @@ jobs:
       x86_64-msvc:
         TOOLCHAIN: stable-x86_64-pc-windows-msvc
         OTHER_TARGET: i686-pc-windows-msvc
+
 - job: rustfmt
   pool:
     vmImage: ubuntu-16.04
@@ -73,6 +74,20 @@ jobs:
   variables:
     TOOLCHAIN: stable
 
+- job: build_std
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+    - template: ci/azure-install-rust.yml
+    - bash: rustup component add rust-src
+      displayName: "Install rust-src"
+    - bash: cargo build
+    - bash: cargo test --test build-std
+      displayName: "tests"
+  variables:
+    TOOLCHAIN: nightly
+    CARGO_RUN_BUILD_STD_TESTS: 1
+
 - job: docs
   pool:
     vmImage: ubuntu-16.04
@@ -90,4 +105,3 @@ jobs:
       displayName: "Build mdbook documentation"
   variables:
     TOOLCHAIN: stable
-

crates/cargo-test-macro/src/lib.rs

@@ -3,7 +3,7 @@ extern crate proc_macro;
 use proc_macro::*;
 
 #[proc_macro_attribute]
-pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
+pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
     let span = Span::call_site();
     let mut ret = TokenStream::new();
     ret.extend(Some(TokenTree::from(Punct::new('#', Spacing::Alone))));
@@ -13,6 +13,8 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
         test.into(),
     ))));
 
+    let build_std = contains_ident(&attr, "build_std");
+
     for token in item {
         let group = match token {
             TokenTree::Group(g) => {
@@ -29,22 +31,20 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
             }
         };
 
-        let mut new_body = vec![
-            TokenTree::from(Ident::new("let", span)),
-            TokenTree::from(Ident::new("_test_guard", span)),
-            TokenTree::from(Punct::new('=', Spacing::Alone)),
-            TokenTree::from(Ident::new("cargo_test_support", span)),
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-            TokenTree::from(Ident::new("paths", span)),
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-            TokenTree::from(Ident::new("init_root", span)),
-            TokenTree::from(Group::new(Delimiter::Parenthesis, TokenStream::new())),
-            TokenTree::from(Punct::new(';', Spacing::Alone)),
-        ]
-        .into_iter()
-        .collect::<TokenStream>();
+        let mut new_body =
+            to_token_stream("let _test_guard = cargo_test_support::paths::init_root();");
+
+        // If this is a `build_std` test (aka `tests/build-std/*.rs`) then they
+        // only run on nightly and they only run when specifically instructed to
+        // on CI.
+        if build_std {
+            let ts = to_token_stream("if !cargo_test_support::is_nightly() { return }");
+            new_body.extend(ts);
+            let ts = to_token_stream(
+                "if std::env::var(\"CARGO_RUN_BUILD_STD_TESTS\").is_err() { return }",
+            );
+            new_body.extend(ts);
+        }
         new_body.extend(group.stream());
         ret.extend(Some(TokenTree::from(Group::new(
             group.delimiter(),
@@ -54,3 +54,14 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
 
     return ret;
 }
+
+fn contains_ident(t: &TokenStream, ident: &str) -> bool {
+    t.clone().into_iter().any(|t| match t {
+        TokenTree::Ident(i) => i.to_string() == ident,
+        _ => false,
+    })
+}
+
+fn to_token_stream(code: &str) -> TokenStream {
+    code.parse().unwrap()
+}

crates/cargo-test-support/Cargo.toml

@@ -4,8 +4,12 @@ version = "0.1.0"
 authors = ["Alex Crichton <alex@alexcrichton.com>"]
 edition = "2018"
 
+[lib]
+doctest = false
+
 [dependencies]
 cargo = { path = "../.." }
+cargo-test-macro = { path = "../cargo-test-macro" }
 filetime = "0.2"
 flate2 = "1.0"
 git2 = "0.10"

crates/cargo-test-support/src/lib.rs

@@ -135,6 +135,8 @@ macro_rules! t {
     };
 }
 
+pub use cargo_test_macro::cargo_test;
+
 pub mod cross_compile;
 pub mod git;
 pub mod paths;

tests/build-std/main.rs

@@ -0,0 +1,180 @@
+//! A test suite for `-Zbuild-std` which is much more expensive than the
+//! standard test suite.
+//!
+//! This test suite attempts to perform a full integration test where we
+//! actually compile the standard library from source (like the real one) and
+//! the various tests associated with that.
+//!
+//! YOU SHOULD IDEALLY NOT WRITE TESTS HERE.
+//!
+//! If possible, use `tests/testsuite/standard_lib.rs` instead. That uses a
+//! 'mock' sysroot which is much faster to compile. The tests here are
+//! extremely intensive and are only intended to run on CI and are theoretically
+//! not catching any regressions that `tests/testsuite/standard_lib.rs` isn't
+//! already catching.
+//!
+//! All tests here should use `#[cargo_test(build_std)]` to indicate that
+//! boilerplate should be generated to require the nightly toolchain and the
+//! `CARGO_RUN_BUILD_STD_TESTS` env var to be set to actually run these tests.
+//! Otherwise the tests are skipped.
+
+use cargo_test_support::*;
+
+fn enable_build_std(e: &mut Execs, arg: Option<&str>) {
+    e.env_remove("CARGO_HOME");
+    e.env_remove("HOME");
+    e.arg("-Zno-index-update");
+
+    // And finally actually enable `build-std` for now
+    let arg = match arg {
+        Some(s) => format!("-Zbuild-std={}", s),
+        None => "-Zbuild-std".to_string(),
+    };
+    e.arg(arg);
+    e.masquerade_as_nightly_cargo();
+}
+
+// Helper methods used in the tests below
+trait BuildStd: Sized {
+    fn build_std(&mut self) -> &mut Self;
+    fn build_std_arg(&mut self, arg: &str) -> &mut Self;
+    fn target_host(&mut self) -> &mut Self;
+}
+
+impl BuildStd for Execs {
+    fn build_std(&mut self) -> &mut Self {
+        enable_build_std(self, None);
+        self
+    }
+
+    fn build_std_arg(&mut self, arg: &str) -> &mut Self {
+        enable_build_std(self, Some(arg));
+        self
+    }
+
+    fn target_host(&mut self) -> &mut Self {
+        self.arg("--target").arg(rustc_host());
+        self
+    }
+}
+
+#[cargo_test(build_std)]
+fn basic() {
+    let p = project()
+        .file(
+            "src/main.rs",
+            "
+                fn main() {
+                    foo::f();
+                }
+
+                #[test]
+                fn smoke_bin_unit() {
+                    foo::f();
+                }
+            ",
+        )
+        .file(
+            "src/lib.rs",
+            "
+                extern crate alloc;
+                extern crate proc_macro;
+
+                /// ```
+                /// foo::f();
+                /// ```
+                pub fn f() {
+                }
+
+                #[test]
+                fn smoke_lib_unit() {
+                    f();
+                }
+            ",
+        )
+        .file(
+            "tests/smoke.rs",
+            "
+                #[test]
+                fn smoke_integration() {
+                    foo::f();
+                }
+            ",
+        )
+        .build();
+
+    p.cargo("check").build_std().target_host().run();
+    p.cargo("build").build_std().target_host().run();
+    p.cargo("run").build_std().target_host().run();
+    p.cargo("test").build_std().target_host().run();
+}
+
+#[cargo_test(build_std)]
+fn cross_custom() {
+    let p = project()
+        .file("src/lib.rs", "#![no_std] pub fn f() {}")
+        .file(
+            "custom-target.json",
+            r#"
+            {
+                "llvm-target": "x86_64-unknown-none-gnu",
+                "data-layout": "e-m:e-i64:64-f80:128-n8:16:32:64-S128",
+                "arch": "x86_64",
+                "target-endian": "little",
+                "target-pointer-width": "64",
+                "target-c-int-width": "32",
+                "os": "none",
+                "linker-flavor": "ld.lld"
+            }
+            "#,
+        )
+        .build();
+
+    p.cargo("build --target custom-target.json -v")
+        .build_std_arg("core")
+        .run();
+}
+
+#[cargo_test(build_std)]
+fn custom_test_framework() {
+    let p = project()
+        .file(
+            "src/lib.rs",
+            r#"
+            #![no_std]
+            #![cfg_attr(test, no_main)]
+            #![feature(custom_test_frameworks)]
+            #![test_runner(crate::test_runner)]
+
+            pub fn test_runner(_tests: &[&dyn Fn()]) {}
+
+            #[panic_handler]
+            fn panic(_info: &core::panic::PanicInfo) -> ! {
+                loop {}
+            }
+            "#,
+        )
+        .file(
+            "target.json",
+            r#"
+            {
+                "llvm-target": "x86_64-unknown-none-gnu",
+                "data-layout": "e-m:e-i64:64-f80:128-n8:16:32:64-S128",
+                "arch": "x86_64",
+                "target-endian": "little",
+                "target-pointer-width": "64",
+                "target-c-int-width": "32",
+                "os": "none",
+                "linker-flavor": "ld.lld",
+                "linker": "rust-lld",
+                "executables": true,
+                "panic-strategy": "abort"
+            }
+            "#,
+        )
+        .build();
+
+    p.cargo("test --target target.json --no-run -v")
+        .build_std_arg("core")
+        .run();
+}

tests/testsuite/standard_lib.rs

@@ -442,74 +442,3 @@ fn doctest() {
 
     p.cargo("test --doc -v").build_std().target_host().run();
 }
-
-// FIXME: set up a dedicated builder to run a full integration test?
-// #[cargo_test]
-// fn cross_custom() {
-//     if !setup() {
-//         return;
-//     }
-//     let p = project()
-//         .file("src/lib.rs", "#![no_std] pub fn f() {}")
-//         .file(
-//             "custom-target.json",
-//             r#"
-//             {
-//                 "llvm-target": "x86_64-unknown-none-gnu",
-//                 "data-layout": "e-m:e-i64:64-f80:128-n8:16:32:64-S128",
-//                 "arch": "x86_64",
-//                 "target-endian": "little",
-//                 "target-pointer-width": "64",
-//                 "target-c-int-width": "32",
-//                 "os": "none",
-//                 "linker-flavor": "ld.lld"
-//             }
-//             "#,
-//         )
-//         .build();
-//
-//     p.cargo("build --target custom-target.json -v")
-//         .build_std()
-//         .run();
-// }
-//
-// fn custom_test_framework() {
-//     let p = project()
-//         .file(
-//             "src/lib.rs",
-//             r#"
-//             #![no_std]
-//             #![cfg_attr(test, no_main)]
-//             #![feature(custom_test_frameworks)]
-//             #![test_runner(crate::test_runner)]
-//
-//             pub fn test_runner(_tests: &[&dyn Fn()]) {}
-//
-//             #[panic_handler]
-//             fn panic(_info: &core::panic::PanicInfo) -> ! {
-//                 loop {}
-//             }
-//             "#,
-//         )
-//         .file(
-//             "target.json",
-//             r#"
-//             {
-//                 "llvm-target": "x86_64-unknown-none-gnu",
-//                 "data-layout": "e-m:e-i64:64-f80:128-n8:16:32:64-S128",
-//                 "arch": "x86_64",
-//                 "target-endian": "little",
-//                 "target-pointer-width": "64",
-//                 "target-c-int-width": "32",
-//                 "os": "none",
-//                 "linker-flavor": "ld.lld",
-//                 "linker": "rust-lld",
-//                 "executables": true,
-//                 "panic-strategy": "abort"
-//             }
-//             "#,
-//         )
-//         .build();
-//
-//     cargo_build_std(&p, "test --target target.json --no-run -v", "core").run();
-// }