Auto merge of rust-lang#111495 - Kobzol:dist-tests, r=Mark-Simulacrum

Run tests on PGO/LTO/BOLT optimized dist artifacts

This PR adds baisc tests for the optimized dist builds on x64 Linux and Windows. A subset of the test suite is run, so it's not perfect, but it's better than the status quo (which is basically no testing at all, apart from the perf bot on Linux).

r? `@ghost`

Author: bors

src/bootstrap/lib.rs

@@ -222,6 +222,7 @@ pub struct Build {
     initial_cargo: PathBuf,
     initial_lld: PathBuf,
     initial_libdir: PathBuf,
+    initial_sysroot: PathBuf,
 
     // Runtime state filled in later on
     // C/C++ compilers and archiver for all targets
@@ -389,13 +390,16 @@ impl Build {
             "/dummy".to_string()
         } else {
             output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot"))
-        };
+        }
+        .trim()
+        .to_string();
+
         let initial_libdir = initial_target_dir
             .parent()
             .unwrap()
             .parent()
             .unwrap()
-            .strip_prefix(initial_sysroot.trim())
+            .strip_prefix(&initial_sysroot)
             .unwrap()
             .to_path_buf();
 
@@ -425,6 +429,7 @@ impl Build {
             initial_cargo: config.initial_cargo.clone(),
             initial_lld,
             initial_libdir,
+            initial_sysroot: initial_sysroot.into(),
             local_rebuild: config.local_rebuild,
             fail_fast: config.cmd.fail_fast(),
             doc_tests: config.cmd.doc_tests(),

src/bootstrap/test.rs

@@ -1424,7 +1424,15 @@ note: if you're sure you want to do this, please open an issue as to why. In the
 
         cmd.arg("--src-base").arg(builder.src.join("tests").join(suite));
         cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
-        cmd.arg("--sysroot-base").arg(builder.sysroot(compiler));
+
+        // When top stage is 0, that means that we're testing an externally provided compiler.
+        // In that case we need to use its specific sysroot for tests to pass.
+        let sysroot = if builder.top_stage == 0 {
+            builder.initial_sysroot.clone()
+        } else {
+            builder.sysroot(compiler).to_path_buf()
+        };
+        cmd.arg("--sysroot-base").arg(sysroot);
         cmd.arg("--stage-id").arg(stage_id);
         cmd.arg("--suite").arg(suite);
         cmd.arg("--mode").arg(mode);

src/ci/stage-build.py

@@ -124,6 +124,12 @@ def llvm_bolt_profile_merged_file(self) -> Path:
     def metrics_path(self) -> Path:
         return self.build_root() / "build" / "metrics.json"
 
+    def executable_extension(self) -> str:
+        raise NotImplementedError
+
+    def skipped_tests(self) -> Iterable[str]:
+        return ()
+
 
 class LinuxPipeline(Pipeline):
     def checkout_path(self) -> Path:
@@ -152,6 +158,13 @@ def build_rustc_perf(self):
     def supports_bolt(self) -> bool:
         return True
 
+    def executable_extension(self) -> str:
+        return ""
+
+    def skipped_tests(self) -> Iterable[str]:
+        # This test fails because of linker errors, as of June 2023.
+        yield "tests/ui/process/nofile-limit.rs"
+
 
 class WindowsPipeline(Pipeline):
     def __init__(self):
@@ -211,6 +224,13 @@ def rustc_profile_template_path(self) -> Path:
     def supports_bolt(self) -> bool:
         return False
 
+    def executable_extension(self) -> str:
+        return ".exe"
+
+    def skipped_tests(self) -> Iterable[str]:
+        # This test fails as of June 2023
+        yield "tests\\codegen\\vec-shrink-panik.rs"
+
 
 def get_timestamp() -> float:
     return time.time()
@@ -403,9 +423,9 @@ def delete_directory(path: Path):
     shutil.rmtree(path)
 
 
-def unpack_archive(archive: Path):
+def unpack_archive(archive: Path, target_dir: Optional[Path] = None):
     LOGGER.info(f"Unpacking archive `{archive}`")
-    shutil.unpack_archive(archive)
+    shutil.unpack_archive(str(archive), extract_dir=str(target_dir) if target_dir is not None else None)
 
 
 def download_file(src: str, target: Path):
@@ -455,6 +475,7 @@ def cmd(
             )
     return subprocess.run(args, env=environment, check=True)
 
+
 class BenchmarkRunner:
     def run_rustc(self, pipeline: Pipeline):
         raise NotImplementedError
@@ -465,6 +486,7 @@ def run_llvm(self, pipeline: Pipeline):
     def run_bolt(self, pipeline: Pipeline):
         raise NotImplementedError
 
+
 class DefaultBenchmarkRunner(BenchmarkRunner):
     def run_rustc(self, pipeline: Pipeline):
         # Here we're profiling the `rustc` frontend, so we also include `Check`.
@@ -478,6 +500,7 @@ def run_rustc(self, pipeline: Pipeline):
                 LLVM_PROFILE_FILE=str(pipeline.rustc_profile_template_path())
             )
         )
+
     def run_llvm(self, pipeline: Pipeline):
         run_compiler_benchmarks(
             pipeline,
@@ -494,6 +517,7 @@ def run_bolt(self, pipeline: Pipeline):
             crates=LLVM_BOLT_CRATES
         )
 
+
 def run_compiler_benchmarks(
         pipeline: Pipeline,
         profiles: List[str],
@@ -650,10 +674,8 @@ def gather_llvm_profiles(pipeline: Pipeline, runner: BenchmarkRunner):
 def gather_rustc_profiles(pipeline: Pipeline, runner: BenchmarkRunner):
     LOGGER.info("Running benchmarks with PGO instrumented rustc")
 
-
     runner.run_rustc(pipeline)
 
-
     profile_path = pipeline.rustc_profile_merged_file()
     LOGGER.info(f"Merging Rustc PGO profiles to {profile_path}")
     cmd([
@@ -770,6 +792,86 @@ def record_metrics(pipeline: Pipeline, timer: Timer):
     log_metrics(metrics)
 
 
+def run_tests(pipeline: Pipeline):
+    """
+    After `dist` is executed, we extract its archived components into a sysroot directory,
+    and then use that extracted rustc as a stage0 compiler.
+    Then we run a subset of tests using that compiler, to have a basic smoke test which checks
+    whether the optimization pipeline hasn't broken something.
+    """
+    build_dir = pipeline.build_root() / "build"
+    dist_dir = build_dir / "dist"
+
+    def extract_dist_dir(name: str) -> Path:
+        target_dir = build_dir / "optimized-dist"
+        target_dir.mkdir(parents=True, exist_ok=True)
+        unpack_archive(dist_dir / f"{name}.tar.xz", target_dir=target_dir)
+        extracted_path = target_dir / name
+        assert extracted_path.is_dir()
+        return extracted_path
+
+    # Extract rustc, libstd, cargo and src archives to create the optimized sysroot
+    rustc_dir = extract_dist_dir(f"rustc-nightly-{PGO_HOST}") / "rustc"
+    libstd_dir = extract_dist_dir(f"rust-std-nightly-{PGO_HOST}") / f"rust-std-{PGO_HOST}"
+    cargo_dir = extract_dist_dir(f"cargo-nightly-{PGO_HOST}") / f"cargo"
+    extracted_src_dir = extract_dist_dir("rust-src-nightly") / "rust-src"
+
+    # We need to manually copy libstd to the extracted rustc sysroot
+    shutil.copytree(
+        libstd_dir / "lib" / "rustlib" / PGO_HOST / "lib",
+        rustc_dir / "lib" / "rustlib" / PGO_HOST / "lib"
+    )
+
+    # Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
+    # sources to the extracted sysroot. We need sources available so that `-Zsimulate-remapped-rust-src-base`
+    # works correctly.
+    shutil.copytree(
+        extracted_src_dir / "lib" / "rustlib" / "src",
+        rustc_dir / "lib" / "rustlib" / "src"
+    )
+
+    rustc_path = rustc_dir / "bin" / f"rustc{pipeline.executable_extension()}"
+    assert rustc_path.is_file()
+    cargo_path = cargo_dir / "bin" / f"cargo{pipeline.executable_extension()}"
+    assert cargo_path.is_file()
+
+    config_content = f"""profile = "user"
+changelog-seen = 2
+
+[build]
+rustc = "{rustc_path.as_posix()}"
+cargo = "{cargo_path.as_posix()}"
+
+[llvm]
+download-ci-llvm = true
+"""
+    logging.info(f"Using following `config.toml` for running tests:\n{config_content}")
+
+    # Simulate a stage 0 compiler with the extracted optimized dist artifacts.
+    with open("config.toml", "w") as f:
+        f.write(config_content)
+
+    args = [
+        sys.executable,
+        pipeline.checkout_path() / "x.py",
+        "test",
+        "--stage", "0",
+        "tests/assembly",
+        "tests/codegen",
+        "tests/codegen-units",
+        "tests/incremental",
+        "tests/mir-opt",
+        "tests/pretty",
+        "tests/run-pass-valgrind",
+        "tests/ui",
+        ]
+    for test_path in pipeline.skipped_tests():
+        args.extend(["--exclude", test_path])
+    cmd(args=args, env=dict(
+        COMPILETEST_FORCE_STAGE0="1"
+    ))
+
+
 def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRunner, final_build_args: List[str]):
     # Clear and prepare tmp directory
     shutil.rmtree(pipeline.opt_artifacts(), ignore_errors=True)
@@ -844,6 +946,11 @@ def execute_build_pipeline(timer: Timer, pipeline: Pipeline, runner: BenchmarkRu
         cmd(final_build_args)
         record_metrics(pipeline, stage4)
 
+    # Try builds can be in various broken states, so we don't want to gatekeep them with tests
+    if not is_try_build():
+        with timer.section("Run tests"):
+            run_tests(pipeline)
+
 
 def run(runner: BenchmarkRunner):
     logging.basicConfig(